ClearML/clearml
1
0
Fork 0
mirror of https://github.com/clearml/clearml synced 2025-06-26 18:16:07 +00:00
Code Issues Actions 1 Packages Projects Releases Wiki Activity

Fix resource monitor fall back to seconds-from-start

Browse Source
This commit is contained in:
allegroai 2019-07-02 00:00:05 +03:00
parent 4099efc26b
commit 738f00340a
1 changed files with 15 additions and 11 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

6
trains/utilities/resource_monitor.py
View File

@ -51,7 +51,7 @@ class ResourceMonitor(object):
last_iteration_ts = 0
last_iteration_interval = None
repeated_iterations = 0
fallback_to_sec_as_iterations = 0
fallback_to_sec_as_iterations = None
while True:
last_report = time()
current_report_frequency = self._report_frequency if reported != 0 else self._first_report_sec
@ -73,6 +73,8 @@ class ResourceMonitor(object):
if IsTensorboardInit.tensorboard_used():
fallback_to_sec_as_iterations = False
elif seconds_since_started >= self._wait_for_first_iteration:
self._task.get_logger().console('TRAINS Monitor: Could not detect iteration reporting, '
'falling back to iterations as seconds-from-start')
fallback_to_sec_as_iterations = True
# if we do not have last_iteration, we just use seconds as iteration
@ -95,6 +97,8 @@ class ResourceMonitor(object):
repeated_iterations = 0
fallback_to_sec_as_iterations = False
# start reporting only when we figured out, if this is seconds based, or iterations based
if fallback_to_sec_as_iterations is not None:
for k, v in average_readouts.items():
# noinspection PyBroadException
try: