ClearML/clearml
1
0
Fork 0
mirror of https://github.com/clearml/clearml synced 2025-05-17 18:54:42 +00:00
Code Issues Actions Packages Projects Releases Wiki Activity

Fix resource monitor and check if task is valid

Browse Source
This commit is contained in:
allegroai 2020-04-09 13:00:24 +03:00
parent 7f00e45d6c
commit 3c4925d605
1 changed files with 8 additions and 3 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

11
trains/utilities/resource_monitor.py
View File

@ -50,7 +50,7 @@ class ResourceMonitor(object):
def start(self): def start(self):
self._exit_event.clear() self._exit_event.clear()
self._thread = Thread(target=self._daemon) self._thread = Thread(target=self._run)
self._thread.daemon = True self._thread.daemon = True
self._thread.start() self._thread.start()
@ -58,8 +58,13 @@ class ResourceMonitor(object):
self._exit_event.set() self._exit_event.set()
# self._thread.join() # self._thread.join()
def _run(self):
try:
self._daemon()
except:
pass
def _daemon(self): def _daemon(self):
logger = self._task.get_logger()
seconds_since_started = 0 seconds_since_started = 0
reported = 0 reported = 0
last_iteration = 0 last_iteration = 0
@ -143,7 +148,7 @@ class ResourceMonitor(object):
title = self._title_gpu if k.startswith('gpu_') else self._title_machine title = self._title_gpu if k.startswith('gpu_') else self._title_machine
# 3 points after the dot # 3 points after the dot
value = round(v*1000) / 1000. value = round(v*1000) / 1000.
logger.report_scalar(title=title, series=k, iteration=iteration, value=value) self._task.get_logger().report_scalar(title=title, series=k, iteration=iteration, value=value)
except Exception: except Exception:
pass pass
# clear readouts if this is update is not averaged # clear readouts if this is update is not averaged