ClearML/clearml
1
0
Fork 0
mirror of https://github.com/clearml/clearml synced 2025-02-07 13:23:40 +00:00
Code Issues Actions Packages Projects Releases Wiki Activity

Fix crash when calling task.flush(wait_for_uploads=True) while executing remotely

Browse Source
This commit is contained in:
allegroai 2022-12-22 22:05:14 +02:00
parent 8992275f8e
commit bc94a55525
1 changed files with 7 additions and 2 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

9
clearml/backend_interface/metrics/reporter.py
View File

@ -108,14 +108,19 @@ class BackgroundReportService(BackgroundMonitor, AsyncManagerMixin):
if isinstance(self._empty_state_event, ForkEvent): if isinstance(self._empty_state_event, ForkEvent):
self._flush_event.set() self._flush_event.set()
tic = time() tic = time()
while self._thread and self._thread.is_alive() and (not timeout or time()-tic < timeout):
while (
self._thread
and (self._thread is True or self._thread.is_alive())
and (not timeout or time() - tic < timeout)
):
if self._empty_state_event.wait(timeout=1.0): if self._empty_state_event.wait(timeout=1.0):
break break
if self._event.wait(0) or self._done_ev.wait(0): if self._event.wait(0) or self._done_ev.wait(0):
break break
# if enough time passed and the flush event was not cleared, # if enough time passed and the flush event was not cleared,
# there is no daemon thread running, we should leave # there is no daemon thread running, we should leave
if time()-tic > self.__daemon_live_check_timeout and self._flush_event.wait(0): if time() - tic > self.__daemon_live_check_timeout and self._flush_event.wait(0):
self._write() self._write()
break break
elif isinstance(self._empty_state_event, SafeEvent): elif isinstance(self._empty_state_event, SafeEvent):