ClearML/clearml
1
0
Fork 0
mirror of https://github.com/clearml/clearml synced 2025-06-23 01:55:38 +00:00
Code Issues Actions Packages Projects Releases Wiki Activity

Fix fork patching, making sure we have a tiny sleep at the end for the signal handler (our at_exit) to be called in time

Browse Source
This commit is contained in:
allegroai 2022-01-08 22:58:08 +02:00
parent afad2772b9
commit fccb14974c
1 changed files with 20 additions and 18 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

38
clearml/binding/environ_bind.py
View File

@ -1,4 +1,5 @@
import os import os
from time import sleep
import six import six
@ -82,26 +83,27 @@ class PatchOsFork(object):
# Make sure the new process stdout is logged # Make sure the new process stdout is logged
if not ret: if not ret:
from ..task import Task from ..task import Task
if Task.current_task() is not None: # force creating a Task
# bind sub-process logger task = Task.current_task()
task = Task.init(project_name=None, task_name=None, task_type=None) # # Hack: now make sure we setup the reporter threads (Log+Reporter)
task.get_logger().flush() if not task._report_subprocess_enabled:
# Hack: now make sure we setup the reporter threads (Log+Reporter)
BackgroundMonitor.start_all(task=task) BackgroundMonitor.start_all(task=task)
# TODO: Check if the signal handler method is enough, for the time being, we have both # The signal handler method is Not enough, for the time being, we have both
# # if we got here patch the os._exit of our instance to call us # even though it makes little sense
def _at_exit_callback(*a_args, **a_kwargs): # # if we got here patch the os._exit of our instance to call us
# call at exit manually def _at_exit_callback(*a_args, **a_kwargs):
# noinspection PyProtectedMember # just make sure we flush the internal state (the at exist caught by the external signal does the rest
task._at_exit() # in theory we should not have to do any of that, but for some reason if we do not
# noinspection PyProtectedMember, PyUnresolvedReferences # the signal is never caught by the signal call backs, not sure why....
return os._org_exit(*a_args, **a_kwargs) sleep(0.1)
# noinspection PyProtectedMember, PyUnresolvedReferences
return os._org_exit(*a_args, **a_kwargs)
if not hasattr(os, '_org_exit'): if not hasattr(os, '_org_exit'):
# noinspection PyProtectedMember, PyUnresolvedReferences # noinspection PyProtectedMember, PyUnresolvedReferences
os._org_exit = os._exit os._org_exit = os._exit
os._exit = _at_exit_callback
os._exit = _at_exit_callback
return ret return ret