ClearML/clearml
1
0
Fork 0
mirror of https://github.com/clearml/clearml synced 2025-03-03 18:52:12 +00:00
Code Issues Actions Packages Projects Releases Wiki Activity

Fix multiple Task.init()/close() calls in the same process

Browse Source
This commit is contained in:
allegroai 2021-02-21 14:59:50 +02:00
parent ee9dfb8b35
commit 132e76cfb9
2 changed files with 21 additions and 2 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

7
clearml/task.py
View File

@ -1327,6 +1327,7 @@ class Task(_Task):
# store is main before we call at_exit, because will will Null it # store is main before we call at_exit, because will will Null it
is_main = self.is_main_task() is_main = self.is_main_task()
is_sub_process = self.__is_subprocess()
# wait for repository detection (5 minutes should be reasonable time to detect all packages) # wait for repository detection (5 minutes should be reasonable time to detect all packages)
if self._logger and not self.__is_subprocess(): if self._logger and not self.__is_subprocess():
@ -1336,6 +1337,11 @@ class Task(_Task):
# unregister atexit callbacks and signal hooks, if we are the main task # unregister atexit callbacks and signal hooks, if we are the main task
if is_main: if is_main:
self.__register_at_exit(None) self.__register_at_exit(None)
if not is_sub_process:
# make sure we enable multiple Task.init callas with reporting sub-processes
BackgroundMonitor.clear_main_process()
# noinspection PyProtectedMember
Logger._remove_std_logger()
def delete(self, delete_artifacts_and_models=True, skip_models_used_by_other_tasks=True, raise_on_error=False): def delete(self, delete_artifacts_and_models=True, skip_models_used_by_other_tasks=True, raise_on_error=False):
# type: (bool, bool, bool) -> bool # type: (bool, bool, bool) -> bool
@ -2975,7 +2981,6 @@ class Task(_Task):
if self.is_main_task(): if self.is_main_task():
Task.__main_task = None Task.__main_task = None
except Exception as ex: except Exception as ex:
import traceback
# make sure we do not interrupt the exit process # make sure we do not interrupt the exit process
pass pass

16
clearml/utilities/process/mp.py
View File

@ -16,7 +16,6 @@ except ImportError: # noqa
class SingletonThreadPool(object): class SingletonThreadPool(object):
__lock = None
__thread_pool = None __thread_pool = None
__thread_pool_pid = None __thread_pool_pid = None
@ -27,6 +26,13 @@ class SingletonThreadPool(object):
cls.__thread_pool_pid = os.getpid() cls.__thread_pool_pid = os.getpid()
return cls.__thread_pool return cls.__thread_pool
@classmethod
def clear(cls):
if cls.__thread_pool:
cls.__thread_pool.close()
cls.__thread_pool = None
cls.__thread_pool_pid = None
class SafeQueue(object): class SafeQueue(object):
__thread_pool = SingletonThreadPool() __thread_pool = SingletonThreadPool()
@ -318,3 +324,11 @@ class BackgroundMonitor(object):
@classmethod @classmethod
def is_subprocess_enabled(cls): def is_subprocess_enabled(cls):
return bool(cls._main_process) return bool(cls._main_process)
@classmethod
def clear_main_process(cls):
BackgroundMonitor._main_process = None
BackgroundMonitor._parent_pid = None
BackgroundMonitor._sub_process_started = None
BackgroundMonitor._instances = {}
SingletonThreadPool.clear()