ClearML/clearml
1
0
Fork 0
mirror of https://github.com/clearml/clearml synced 2025-03-03 18:52:12 +00:00
Code Issues Actions Packages Projects Releases Wiki Activity

Fix flush (wait) on auxiliary task (obtained using Task.get_task()) should wait on all upload events

Browse Source
This commit is contained in:
allegroai 2020-12-06 11:25:20 +02:00
parent 73540fbb62
commit 5e70a9e6eb
2 changed files with 11 additions and 0 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

8
trains/backend_interface/metrics/reporter.py
View File

@ -1,6 +1,7 @@
import json import json
import logging import logging
import math import math
from time import time
try: try:
from collections.abc import Iterable # noqa from collections.abc import Iterable # noqa
@ -135,6 +136,13 @@ class Reporter(InterfaceBase, AbstractContextManager, SetupUploadMixin, AsyncMan
self._flush_event.set() self._flush_event.set()
self._thread.join() self._thread.join()
def wait_for_events(self, timeout=None, step=2.0):
tic = time()
while self._events or self.get_num_results():
self.wait_for_results(timeout=step)
if timeout and time() - tic >= timeout:
break
def report_scalar(self, title, series, value, iter): def report_scalar(self, title, series, value, iter):
""" """
Report a scalar value Report a scalar value

3
trains/task.py
View File

@ -1215,6 +1215,9 @@ class Task(_Task):
self._logger._flush_stdout_handler() self._logger._flush_stdout_handler()
if self.__reporter: if self.__reporter:
self.__reporter.flush() self.__reporter.flush()
if wait_for_uploads:
self.__reporter.wait_for_events()
LoggerRoot.flush() LoggerRoot.flush()
return True return True