ClearML/clearml
1
0
Fork 0
mirror of https://github.com/clearml/clearml synced 2025-03-03 18:52:12 +00:00
Code Issues Actions Packages Projects Releases Wiki Activity

Wait for subprocess when terminating

Browse Source
This commit is contained in:
allegroai 2021-04-20 18:08:22 +03:00
parent 1c6685e273
commit 4832de5a65
2 changed files with 117 additions and 108 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
clearml/task.py
View File

@ -2886,7 +2886,6 @@ class Task(_Task):
is_sub_process = self.__is_subprocess()
if True: # not is_sub_process: # todo: remove IF
# noinspection PyBroadException
try:
wait_for_uploads = True
@ -3030,6 +3029,7 @@ class Task(_Task):
# make sure no one will re-enter the shutdown method
self._at_exit_called = True
BackgroundMonitor.wait_for_sub_process()
@classmethod
def __register_at_exit(cls, exit_callback, only_remove_signal_and_exception_hooks=False):

11
clearml/utilities/process/mp.py
View File

@ -6,7 +6,7 @@ from functools import partial
from multiprocessing import Process, Lock, Event as ProcessEvent
from multiprocessing.pool import ThreadPool
from threading import Thread, Event as TrEvent
from time import sleep
from time import sleep, time
from typing import List, Dict
import psutil
@ -415,8 +415,17 @@ class BackgroundMonitor(object):
@classmethod
def clear_main_process(cls):
cls.wait_for_sub_process()
BackgroundMonitor._main_process = None
BackgroundMonitor._parent_pid = None
BackgroundMonitor._sub_process_started = None
BackgroundMonitor._instances = {}
SingletonThreadPool.clear()
@classmethod
def wait_for_sub_process(cls, timeout=None):
if not cls.is_subprocess_enabled():
return
tic = time()
while cls.is_subprocess_alive() and (not timeout or time()-tic < timeout):
sleep(0.03)