ClearML/clearml
1
0
Fork 0
mirror of https://github.com/clearml/clearml synced 2025-06-04 03:47:57 +00:00
Code Issues Actions Packages Projects Releases Wiki Activity

Wait for subprocess when terminating

Browse Source
This commit is contained in:
allegroai 2021-04-20 18:08:22 +03:00
parent 1c6685e273
commit 4832de5a65
2 changed files with 117 additions and 108 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
clearml/task.py
View File

@ -2886,7 +2886,6 @@ class Task(_Task):
is_sub_process = self.__is_subprocess() is_sub_process = self.__is_subprocess()
if True: # not is_sub_process: # todo: remove IF
# noinspection PyBroadException # noinspection PyBroadException
try: try:
wait_for_uploads = True wait_for_uploads = True
@ -3030,6 +3029,7 @@ class Task(_Task):
# make sure no one will re-enter the shutdown method # make sure no one will re-enter the shutdown method
self._at_exit_called = True self._at_exit_called = True
BackgroundMonitor.wait_for_sub_process()
@classmethod @classmethod
def __register_at_exit(cls, exit_callback, only_remove_signal_and_exception_hooks=False): def __register_at_exit(cls, exit_callback, only_remove_signal_and_exception_hooks=False):

11
clearml/utilities/process/mp.py
View File

@ -6,7 +6,7 @@ from functools import partial
from multiprocessing import Process, Lock, Event as ProcessEvent from multiprocessing import Process, Lock, Event as ProcessEvent
from multiprocessing.pool import ThreadPool from multiprocessing.pool import ThreadPool
from threading import Thread, Event as TrEvent from threading import Thread, Event as TrEvent
from time import sleep from time import sleep, time
from typing import List, Dict from typing import List, Dict
import psutil import psutil
@ -415,8 +415,17 @@ class BackgroundMonitor(object):
@classmethod @classmethod
def clear_main_process(cls): def clear_main_process(cls):
cls.wait_for_sub_process()
BackgroundMonitor._main_process = None BackgroundMonitor._main_process = None
BackgroundMonitor._parent_pid = None BackgroundMonitor._parent_pid = None
BackgroundMonitor._sub_process_started = None BackgroundMonitor._sub_process_started = None
BackgroundMonitor._instances = {} BackgroundMonitor._instances = {}
SingletonThreadPool.clear() SingletonThreadPool.clear()
@classmethod
def wait_for_sub_process(cls, timeout=None):
if not cls.is_subprocess_enabled():
return
tic = time()
while cls.is_subprocess_alive() and (not timeout or time()-tic < timeout):
sleep(0.03)