ClearML/clearml-agent
1
0
Fork 0
mirror of https://github.com/clearml/clearml-agent synced 2025-06-26 18:16:15 +00:00
Code Issues Actions Packages Projects Releases Wiki Activity

Print error on resource monitor failure

Browse Source
This commit is contained in:
allegroai 2023-05-11 16:18:11 +03:00
parent 307ec9213e
commit b6ca0fa6a5
1 changed files with 34 additions and 31 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

3
clearml_agent/helper/resource_monitor.py
View File

@ -139,6 +139,7 @@ class ResourceMonitor(object):
def _daemon(self): def _daemon(self):
seconds_since_started = 0 seconds_since_started = 0
reported = 0 reported = 0
try:
while True: while True:
last_report = time() last_report = time()
current_report_frequency = ( current_report_frequency = (
@ -175,6 +176,8 @@ class ResourceMonitor(object):
# count reported iterations # count reported iterations
reported += 1 reported += 1
except Exception as ex:
log.exception("Error reporting monitoring info: %s", str(ex))
def _update_readouts(self): def _update_readouts(self):
readouts = self._machine_stats() readouts = self._machine_stats()