ClearML/clearml
1
0
Fork 0
mirror of https://github.com/clearml/clearml synced 2025-06-23 01:55:38 +00:00
Code Issues Actions Packages Projects Releases Wiki Activity

Fix multiple tasks closing

Browse Source
This commit is contained in:
allegroai 2019-09-15 01:36:33 +03:00
parent 61fb17266f
commit 8c3f6d254a
1 changed files with 12 additions and 7 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

11
trains/task.py
View File

@ -615,6 +615,7 @@ class Task(_Task):
if self._logger: if self._logger:
# noinspection PyProtectedMember # noinspection PyProtectedMember
self._logger._flush_stdout_handler() self._logger._flush_stdout_handler()
if self._reporter:
self.reporter.flush() self.reporter.flush()
LoggerRoot.flush() LoggerRoot.flush()
@ -1045,7 +1046,7 @@ class Task(_Task):
task_status = ('stopped', ) task_status = ('stopped', )
# wait for repository detection (if we didn't crash) # wait for repository detection (if we didn't crash)
if not is_sub_process and wait_for_uploads: if not is_sub_process and wait_for_uploads and self._logger:
# we should print summary here # we should print summary here
self._summary_artifacts() self._summary_artifacts()
# make sure that if we crashed the thread we are not waiting forever # make sure that if we crashed the thread we are not waiting forever
@ -1053,17 +1054,19 @@ class Task(_Task):
# wait for uploads # wait for uploads
print_done_waiting = False print_done_waiting = False
if wait_for_uploads and (BackendModel.get_num_results() > 0 or self.reporter.get_num_results() > 0): if wait_for_uploads and (BackendModel.get_num_results() > 0 or
(self._reporter and self.reporter.get_num_results() > 0)):
self.log.info('Waiting to finish uploads') self.log.info('Waiting to finish uploads')
print_done_waiting = True print_done_waiting = True
# from here, do not send log in background thread # from here, do not send log in background thread
if wait_for_uploads: if wait_for_uploads:
self.flush(wait_for_uploads=True) self.flush(wait_for_uploads=True)
# wait until the reporter flush everything # wait until the reporter flush everything
if self._reporter:
self.reporter.stop() self.reporter.stop()
if print_done_waiting: if print_done_waiting:
self.log.info('Finished uploading') self.log.info('Finished uploading')
else: elif self._logger:
self._logger._flush_stdout_handler() self._logger._flush_stdout_handler()
if not is_sub_process: if not is_sub_process:
@ -1085,8 +1088,10 @@ class Task(_Task):
if self._resource_monitor: if self._resource_monitor:
self._resource_monitor.stop() self._resource_monitor.stop()
if self._logger:
self._logger.set_flush_period(None) self._logger.set_flush_period(None)
# this is so in theory we can close a main task and start a new one # this is so in theory we can close a main task and start a new one
if self.is_main_task():
Task.__main_task = None Task.__main_task = None
except Exception: except Exception:
# make sure we do not interrupt the exit process # make sure we do not interrupt the exit process