ClearML/clearml
1
0
Fork 0
mirror of https://github.com/clearml/clearml synced 2025-03-03 10:42:00 +00:00
Code Issues Actions Packages Projects Releases Wiki Activity

Retry sending console logs if session.send() fails (applicable only in local mode where we use the logging handler)

Browse Source
This commit is contained in:
allegroai 2020-02-26 17:07:07 +02:00
parent cf850020fb
commit 8ee2bd1844
1 changed files with 4 additions and 2 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

6
trains/backend_interface/task/log.py
View File

@ -128,5 +128,7 @@ class TaskHandler(BufferingHandler):
res = self.session.send(a_request)
if not res.ok():
print("Failed logging task to backend ({:d} lines, {})".format(len(a_request.requests), str(res.meta)))
except Exception:
print("Failed logging task to backend ({:d} lines)".format(len(a_request.requests)))
except Exception as ex:
print("Retrying, failed logging task to backend ({:d} lines): {}".format(len(a_request.requests), ex))
# we should push ourselves back into the thread pool
self._thread_pool.apply_async(self._send_events, args=(a_request, ))