ClearML/clearml
1
0
Fork 0
mirror of https://github.com/clearml/clearml synced 2025-03-04 19:20:16 +00:00
Code Issues Actions Packages Projects Releases Wiki Activity

Fix multi-subprocess can cause Task to hand at close

Browse Source
This commit is contained in:
allegroai 2021-11-14 22:25:44 +02:00
parent f52fcb9668
commit f75a210335
1 changed files with 14 additions and 2 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

16
clearml/backend_interface/metrics/reporter.py
View File

@ -4,7 +4,7 @@ import logging
import math import math
from multiprocessing import Semaphore from multiprocessing import Semaphore
from threading import Event as TrEvent from threading import Event as TrEvent
from time import sleep from time import sleep, time
import numpy as np import numpy as np
import six import six
@ -76,8 +76,20 @@ class BackgroundReportService(BackgroundMonitor, AsyncManagerMixin):
while self._queue and not self._queue.empty(): while self._queue and not self._queue.empty():
sleep(0.1) sleep(0.1)
return return
self._empty_state_event.clear() self._empty_state_event.clear()
return self._empty_state_event.wait(timeout) if isinstance(self._empty_state_event, TrEvent):
tic = time()
while self._thread and self._thread.is_alive() and (not timeout or time()-tic < timeout):
if self._empty_state_event.wait(timeout=1.0):
break
elif isinstance(self._empty_state_event, SafeEvent):
tic = time()
while self.is_subprocess_alive() and (not timeout or time()-tic < timeout):
if self._empty_state_event.wait(timeout=1.0):
break
return
def add_event(self, ev): def add_event(self, ev):
if not self._queue: if not self._queue: