From ab2ab50e4c3d5955d9c95b78cde5ba959c785b6c Mon Sep 17 00:00:00 2001 From: allegroai <> Date: Tue, 17 May 2022 14:52:54 +0300 Subject: [PATCH] Fix Process Pool hangs at exit (#674) --- clearml/binding/environ_bind.py | 3 ++- clearml/task.py | 4 +--- clearml/version.py | 2 +- 3 files changed, 4 insertions(+), 5 deletions(-) diff --git a/clearml/binding/environ_bind.py b/clearml/binding/environ_bind.py index 8d6ce306..2c5257f6 100644 --- a/clearml/binding/environ_bind.py +++ b/clearml/binding/environ_bind.py @@ -1,4 +1,5 @@ import os +from time import sleep import six @@ -106,7 +107,7 @@ class PatchOsFork(object): # just make sure we flush the internal state (the at exist caught by the external signal does the rest # in theory we should not have to do any of that, but for some reason if we do not # the signal is never caught by the signal call backs, not sure why.... - + sleep(0.1) # Since at_exist handlers do not work on forked processes, we have to manually call them here if task: try: diff --git a/clearml/task.py b/clearml/task.py index ebfd19c9..e4239463 100644 --- a/clearml/task.py +++ b/clearml/task.py @@ -3460,6 +3460,7 @@ class Task(_Task): elif task_status[0] == 'failed': self.mark_failed(status_reason=task_status[1]) elif task_status[0] == 'completed': + self.set_progress(100) self.mark_completed() elif task_status[0] == 'stopped': self.stopped() @@ -3497,9 +3498,6 @@ class Task(_Task): pass self._edit_lock = None - if task_status and task_status[0] == "completed": - self.set_progress(100) - # make sure no one will re-enter the shutdown method self._at_exit_called = True if not is_sub_process and BackgroundMonitor.is_subprocess_enabled(): diff --git a/clearml/version.py b/clearml/version.py index 96e3ce8d..8e3c933c 100644 --- a/clearml/version.py +++ b/clearml/version.py @@ -1 +1 @@ -__version__ = '1.4.0' +__version__ = '1.4.1'