ClearML/clearml
1
0
Fork 0
mirror of https://github.com/clearml/clearml synced 2025-05-22 21:14:04 +00:00
Code Issues Actions Packages Projects Releases Wiki Activity

Add 10sec timeout for stdout/stderr flush at end of process

Browse Source
This commit is contained in:
allegroai 2020-04-09 12:46:30 +03:00
parent 1718aa20d4
commit 9916c93ce0
1 changed files with 10 additions and 3 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

13
trains/backend_interface/task/log.py
View File

@ -12,6 +12,7 @@ buffer_capacity = config.get('log.task_log_buffer_capacity', 100)
class TaskHandler(BufferingHandler): class TaskHandler(BufferingHandler):
__flush_max_history_seconds = 30. __flush_max_history_seconds = 30.
__wait_for_flush_timeout = int(10)
__once = False __once = False
@property @property
@ -130,6 +131,7 @@ class TaskHandler(BufferingHandler):
def wait_for_flush(self, shutdown=False): def wait_for_flush(self, shutdown=False):
msg = 'Task.log.wait_for_flush: %d' msg = 'Task.log.wait_for_flush: %d'
from ...utilities.os.lowlevel import threadpool_waited_join
ll = self.__log_stderr ll = self.__log_stderr
ll(msg % 0) ll(msg % 0)
self.acquire() self.acquire()
@ -142,9 +144,14 @@ class TaskHandler(BufferingHandler):
ll(msg % 4) ll(msg % 4)
try: try:
ll(msg % 5) ll(msg % 5)
t.close() t.terminate()
ll(msg % 6) ll(msg % 6)
t.join() for i in range(int(self.__wait_for_flush_timeout)):
if self._pending <= 0:
break
self.__log_stderr('INFO: trains.Task - flushing console logs (timeout {}s)'.format(
self.__wait_for_flush_timeout-i))
threadpool_waited_join(t, timeout=1.0)
ll(msg % 7) ll(msg % 7)
except Exception: except Exception:
ll(msg % 8) ll(msg % 8)
@ -175,7 +182,7 @@ class TaskHandler(BufferingHandler):
try: try:
if self._thread_pool is None: if self._thread_pool is None:
self.__log_stderr('INFO: trains.Task - ' self.__log_stderr('INFO: trains.Task - '
'Task.close() flushing remaining logs ({}){}'.format(self._pending, a_request)) 'Task.close() flushing remaining logs ({})'.format(self._pending))
self._pending -= 1 self._pending -= 1
res = self.session.send(a_request) res = self.session.send(a_request)
if not res.ok(): if not res.ok():