ClearML/clearml
1
0
Fork 0
mirror of https://github.com/clearml/clearml synced 2025-02-12 07:35:08 +00:00
Code Issues Actions Packages Projects Releases Wiki Activity

Fix python3.8 race condition in task.close()

Browse Source
This commit is contained in:
allegroai 2021-05-10 20:33:30 +03:00
parent 3225715ae3
commit 0e0d86f6d8
2 changed files with 9 additions and 5 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
clearml/task.py
View File

@ -1396,7 +1396,7 @@ class Task(_Task):
self.__register_at_exit(None) self.__register_at_exit(None)
if not is_sub_process: if not is_sub_process:
# make sure we enable multiple Task.init callas with reporting sub-processes # make sure we enable multiple Task.init callas with reporting sub-processes
BackgroundMonitor.clear_main_process() BackgroundMonitor.clear_main_process(self)
# noinspection PyProtectedMember # noinspection PyProtectedMember
Logger._remove_std_logger() Logger._remove_std_logger()
@ -3050,7 +3050,7 @@ class Task(_Task):
# make sure no one will re-enter the shutdown method # make sure no one will re-enter the shutdown method
self._at_exit_called = True self._at_exit_called = True
BackgroundMonitor.wait_for_sub_process() BackgroundMonitor.wait_for_sub_process(self)
@classmethod @classmethod
def __register_at_exit(cls, exit_callback, only_remove_signal_and_exception_hooks=False): def __register_at_exit(cls, exit_callback, only_remove_signal_and_exception_hooks=False):

10
clearml/utilities/process/mp.py
View File

@ -414,8 +414,8 @@ class BackgroundMonitor(object):
return bool(cls._main_process) return bool(cls._main_process)
@classmethod @classmethod
def clear_main_process(cls): def clear_main_process(cls, task):
cls.wait_for_sub_process() cls.wait_for_sub_process(task)
BackgroundMonitor._main_process = None BackgroundMonitor._main_process = None
BackgroundMonitor._parent_pid = None BackgroundMonitor._parent_pid = None
BackgroundMonitor._sub_process_started = None BackgroundMonitor._sub_process_started = None
@ -423,9 +423,13 @@ class BackgroundMonitor(object):
SingletonThreadPool.clear() SingletonThreadPool.clear()
@classmethod @classmethod
def wait_for_sub_process(cls, timeout=None): def wait_for_sub_process(cls, task, timeout=None):
if not cls.is_subprocess_enabled(): if not cls.is_subprocess_enabled():
return return
for d in BackgroundMonitor._instances.get(id(task.id), []):
d.stop()
tic = time() tic = time()
while cls.is_subprocess_alive() and (not timeout or time()-tic < timeout): while cls.is_subprocess_alive() and (not timeout or time()-tic < timeout):
sleep(0.03) sleep(0.03)