ClearML/clearml
1
0
Fork 0
mirror of https://github.com/clearml/clearml synced 2025-05-08 14:54:28 +00:00
Code Issues Actions Packages Projects Releases Wiki Activity

Fix plot reporting for models

Browse Source
This commit is contained in:
Alex Burlacu 2023-04-03 18:36:45 +03:00
parent 4ad4939feb
commit d564940cd4
1 changed files with 23 additions and 6 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

29
clearml/backend_interface/metrics/reporter.py
View File

@ -1,3 +1,4 @@
import atexit
import datetime import datetime
import json import json
import logging import logging
@ -278,6 +279,7 @@ class Reporter(InterfaceBase, AbstractContextManager, SetupUploadMixin, AsyncMan
flush_threshold=flush_threshold, flush_threshold=flush_threshold,
for_model=for_model, for_model=for_model,
) )
atexit.register(self._handle_program_exit)
self._report_service.start() self._report_service.start()
def _set_storage_uri(self, value): def _set_storage_uri(self, value):
@ -309,21 +311,36 @@ class Reporter(InterfaceBase, AbstractContextManager, SetupUploadMixin, AsyncMan
self._max_iteration = max(self._max_iteration, ev_iteration + self._metrics.get_iteration_offset()) self._max_iteration = max(self._max_iteration, ev_iteration + self._metrics.get_iteration_offset())
self._report_service.add_event(ev) self._report_service.add_event(ev)
def _handle_program_exit(self):
try:
self.flush()
self.wait_for_events()
self.stop()
except Exception as e:
logging.getLogger("clearml.reporter").warning(
"Exception encountered cleaning up the reporter: {}".format(e)
)
def flush(self): def flush(self):
""" """
Flush cached reports to backend. Flush cached reports to backend.
""" """
if self._report_service: # we copy this value for thread safety
self._report_service.flush() report_service = self._report_service
if report_service:
report_service.flush()
def wait_for_events(self, timeout=None): def wait_for_events(self, timeout=None):
if self._report_service: # we copy this value for thread safety
return self._report_service.wait_for_events(timeout=timeout) report_service = self._report_service
if report_service:
return report_service.wait_for_events(timeout=timeout)
def stop(self): def stop(self):
if not self._report_service: # save the report service and allow multiple threads to access it
return
report_service = self._report_service report_service = self._report_service
if not report_service:
return
self._report_service = None self._report_service = None
if not report_service.is_subprocess_mode() or report_service.is_alive(): if not report_service.is_subprocess_mode() or report_service.is_alive():
report_service.stop() report_service.stop()