ClearML/clearml
1
0
Fork 0
mirror of https://github.com/clearml/clearml synced 2025-03-04 02:57:24 +00:00
Code Issues Actions Packages Projects Releases Wiki Activity

Use sub-process fork signal hooks instead of os._exit patch

Browse Source
This commit is contained in:
allegroai 2020-04-09 12:50:04 +03:00
parent 004f925454
commit 2f395cc76b
2 changed files with 16 additions and 12 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

22
trains/binding/environ_bind.py
View File

@ -68,16 +68,18 @@ class PatchOsFork(object):
# Hack: now make sure we setup the reporter thread # Hack: now make sure we setup the reporter thread
task._setup_reporter() task._setup_reporter()
# if we got here patch the os._exit of our instance to call us
def _at_exit_callback(*args, **kwargs):
# call at exit manually
# noinspection PyProtectedMember
task._at_exit()
# noinspection PyProtectedMember
return os._org_exit(*args, **kwargs)
if not hasattr(os, '_org_exit'): # TODO: Check if the signal handler method is enough, for the time being, disable the _exit hook
os._org_exit = os._exit # # if we got here patch the os._exit of our instance to call us
os._exit = _at_exit_callback # def _at_exit_callback(*args, **kwargs):
# # call at exit manually
# # noinspection PyProtectedMember
# task._at_exit()
# # noinspection PyProtectedMember
# return os._org_exit(*args, **kwargs)
#
# if not hasattr(os, '_org_exit'):
# os._org_exit = os._exit
# os._exit = _at_exit_callback
return ret return ret

6
trains/task.py
View File

@ -218,7 +218,8 @@ class Task(_Task):
cls.__main_task._artifacts_manager = Artifacts(cls.__main_task) cls.__main_task._artifacts_manager = Artifacts(cls.__main_task)
# unregister signal hooks, they cause subprocess to hang # unregister signal hooks, they cause subprocess to hang
cls.__main_task.__register_at_exit(cls.__main_task._at_exit) cls.__main_task.__register_at_exit(cls.__main_task._at_exit)
cls.__main_task.__register_at_exit(None, only_remove_signal_and_exception_hooks=True) # TODO: Check if the signal handler method is safe enough, for the time being, do not unhook
# cls.__main_task.__register_at_exit(None, only_remove_signal_and_exception_hooks=True)
if not running_remotely(): if not running_remotely():
verify_defaults_match() verify_defaults_match()
@ -1633,7 +1634,8 @@ class Task(_Task):
if self._exit_callback: if self._exit_callback:
atexit.register(self._exit_callback) atexit.register(self._exit_callback)
if not self._org_handlers and not Task._Task__is_subprocess(): # TODO: check if sub-process hooks are safe enough, for the time being allow it
if not self._org_handlers: # ## and not Task._Task__is_subprocess():
if sys.platform == 'win32': if sys.platform == 'win32':
catch_signals = [signal.SIGINT, signal.SIGTERM, signal.SIGSEGV, signal.SIGABRT, catch_signals = [signal.SIGINT, signal.SIGTERM, signal.SIGSEGV, signal.SIGABRT,
signal.SIGILL, signal.SIGFPE] signal.SIGILL, signal.SIGFPE]