ClearML/clearml-agent
1
0
Fork 0
mirror of https://github.com/clearml/clearml-agent synced 2025-06-08 23:47:15 +00:00
Code Issues Actions Packages Projects Releases Wiki Activity

Make sure logs are sent even in case an exception occurs inside the logging monitor

Browse Source
This commit is contained in:
allegroai 2020-01-13 18:14:12 +02:00
parent a9f479cfcd
commit 7a11c7c165
1 changed files with 12 additions and 2 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

12
trains_agent/commands/worker.py
View File

@ -753,9 +753,9 @@ class Worker(ServiceCommandSection):
stdout = open(stdout_path, "wt") stdout = open(stdout_path, "wt")
stderr = open(stderr_path, "wt") if stderr_path else stdout stderr = open(stderr_path, "wt") if stderr_path else stdout
try:
stdout_line_count, stdout_last_lines = 0, [] stdout_line_count, stdout_last_lines = 0, []
stderr_line_count, stderr_last_lines = 0, [] stderr_line_count, stderr_last_lines = 0, []
try:
status = None status = None
stopping = False stopping = False
_last_machine_update_ts = time() _last_machine_update_ts = time()
@ -803,6 +803,16 @@ class Worker(ServiceCommandSection):
# non zero return code # non zero return code
stop_reason = 'Exception occurred' stop_reason = 'Exception occurred'
status = ex.returncode status = ex.returncode
except KeyboardInterrupt:
# so someone else will catch us
raise
except Exception:
# we should not get here, but better safe than sorry
stdout_line_count += self.send_logs(task_id, _print_file(stdout_path, stdout_line_count))
if stderr_path:
stderr_line_count += self.send_logs(task_id, _print_file(stderr_path, stderr_line_count))
stop_reason = 'Exception occurred'
status = -1
stdout.close() stdout.close()
if stderr_path: if stderr_path: