ClearML/clearml
1
0
Fork 0
mirror of https://github.com/clearml/clearml synced 2025-02-07 13:23:40 +00:00
Code Issues Actions Packages Projects Releases Wiki Activity

Fix plot reporting for models

Browse Source
This commit is contained in:
Alex Burlacu 2023-04-03 18:36:45 +03:00
parent 4ad4939feb
commit d564940cd4
1 changed files with 23 additions and 6 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

29
clearml/backend_interface/metrics/reporter.py
View File

@ -1,3 +1,4 @@
import atexit
import datetime
import json
import logging
@ -278,6 +279,7 @@ class Reporter(InterfaceBase, AbstractContextManager, SetupUploadMixin, AsyncMan
flush_threshold=flush_threshold,
for_model=for_model,
)
atexit.register(self._handle_program_exit)
self._report_service.start()
def _set_storage_uri(self, value):
@ -309,21 +311,36 @@ class Reporter(InterfaceBase, AbstractContextManager, SetupUploadMixin, AsyncMan
self._max_iteration = max(self._max_iteration, ev_iteration + self._metrics.get_iteration_offset())
self._report_service.add_event(ev)
def _handle_program_exit(self):
try:
self.flush()
self.wait_for_events()
self.stop()
except Exception as e:
logging.getLogger("clearml.reporter").warning(
"Exception encountered cleaning up the reporter: {}".format(e)
)
def flush(self):
"""
Flush cached reports to backend.
"""
if self._report_service:
self._report_service.flush()
# we copy this value for thread safety
report_service = self._report_service
if report_service:
report_service.flush()
def wait_for_events(self, timeout=None):
if self._report_service:
return self._report_service.wait_for_events(timeout=timeout)
# we copy this value for thread safety
report_service = self._report_service
if report_service:
return report_service.wait_for_events(timeout=timeout)
def stop(self):
if not self._report_service:
return
# save the report service and allow multiple threads to access it
report_service = self._report_service
if not report_service:
return
self._report_service = None
if not report_service.is_subprocess_mode() or report_service.is_alive():
report_service.stop()