ClearML/clearml
1
0
Fork 0
mirror of https://github.com/clearml/clearml synced 2025-06-03 11:27:00 +00:00
Code Issues Actions 1 Packages Projects Releases Wiki Activity

Stop resource monitoring before signaling task stop

Browse Source
This commit is contained in:
allegroai 2020-04-09 13:07:26 +03:00
parent ab263bb59f
commit d9aa83380f
1 changed files with 9 additions and 6 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

15
trains/task.py
View File

@ -1559,6 +1559,11 @@ class Task(_Task):
self._dev_worker.unregister() self._dev_worker.unregister()
self._dev_worker = None self._dev_worker = None
# stop resource monitoring
if self._resource_monitor:
self._resource_monitor.stop()
self._resource_monitor = None
if not is_sub_process: if not is_sub_process:
# change task status # change task status
if not task_status: if not task_status:
@ -1570,10 +1575,6 @@ class Task(_Task):
elif task_status[0] == 'stopped': elif task_status[0] == 'stopped':
self.stopped() self.stopped()
# stop resource monitoring
if self._resource_monitor:
self._resource_monitor.stop()
if self._logger: if self._logger:
self._logger.set_flush_period(None) self._logger.set_flush_period(None)
self._logger._close_stdout_handler(wait=wait_for_uploads or wait_for_std_log) self._logger._close_stdout_handler(wait=wait_for_uploads or wait_for_std_log)
@ -1634,10 +1635,12 @@ class Task(_Task):
def hook(self): def hook(self):
if self._orig_exit is None: if self._orig_exit is None:
self._orig_exit = sys.exit self._orig_exit = sys.exit
sys.exit = self.exit sys.exit = self.exit
if self._orig_exc_handler is None: if self._orig_exc_handler is None:
self._orig_exc_handler = sys.excepthook self._orig_exc_handler = sys.excepthook
sys.excepthook = self.exc_handler sys.excepthook = self.exc_handler
if self._exit_callback: if self._exit_callback:
atexit.register(self._exit_callback) atexit.register(self._exit_callback)