diff --git a/clearml/backend_interface/metrics/interface.py b/clearml/backend_interface/metrics/interface.py index a029831a..30544008 100644 --- a/clearml/backend_interface/metrics/interface.py +++ b/clearml/backend_interface/metrics/interface.py @@ -222,7 +222,13 @@ class Metrics(InterfaceBase): if batched_requests: if self._offline_mode: with open(self._offline_log_filename.as_posix(), 'at') as f: - f.write(json.dumps([b.to_dict() for b in batched_requests])+'\n') + requests = [] + for b in batched_requests: + request = b.to_dict() + if self._for_model: + request["model_event"] = True + requests.append(request) + f.write(json.dumps(requests)+'\n') return req = api_events.AddBatchRequest(requests=batched_requests) @@ -262,13 +268,16 @@ class Metrics(InterfaceBase): pass @classmethod - def report_offline_session(cls, task, folder, iteration_offset=0): + def report_offline_session(cls, task, folder, iteration_offset=0, remote_url=None, only_with_id=None, session=None): from ... import StorageManager filename = Path(folder) / cls.__offline_filename if not filename.is_file(): return False - # noinspection PyProtectedMember - remote_url = task._get_default_report_storage_uri() + if not remote_url: + # noinspection PyProtectedMember + remote_url = task._get_default_report_storage_uri() + if not session: + session = task.session if remote_url and remote_url.endswith('/'): remote_url = remote_url[:-1] uploaded_files = set() @@ -281,6 +290,8 @@ class Metrics(InterfaceBase): if not line: break list_requests = json.loads(line) + if only_with_id: + list_requests = [r for r in list_requests if r["task"] == only_with_id] for r in list_requests: # noinspection PyBroadException try: @@ -327,7 +338,7 @@ class Metrics(InterfaceBase): warning('Failed reporting metric, line {} [{}]'.format(i, ex)) batch_requests = api_events.AddBatchRequest(requests=list_requests) if batch_requests.requests: - res = task.session.send(batch_requests) + res = session.send(batch_requests) if res and not res.ok(): warning("failed logging metric task to backend ({:d} lines, {})".format( len(batch_requests.requests), str(res.meta))) diff --git a/clearml/model.py b/clearml/model.py index 81335e1d..877ef60e 100644 --- a/clearml/model.py +++ b/clearml/model.py @@ -3,6 +3,7 @@ import os import zipfile import shutil from tempfile import mkstemp +from uuid import uuid4 import six import math @@ -1070,6 +1071,10 @@ class BaseModel(object): except Exception: pass + def get_offline_mode_folder(self): + from clearml import Task as OfflineTask + return OfflineTask.current_task().get_offline_mode_folder() + def _init_reporter(self): if self._reporter: return @@ -2154,6 +2159,16 @@ class OutputModel(BaseModel): """ return self._get_base_model().upload_storage_uri + @property + def id(self): + # type: () -> str + from clearml import Task as OfflineTask + if OfflineTask.is_offline(): + if not self._base_model_id: + self._base_model_id = "offline-{}".format(str(uuid4()).replace("-", "")) + return self._base_model_id + return super(OutputModel, self).id + def __init__( self, task=None, # type: Optional[Task] @@ -2810,7 +2825,8 @@ class OutputModel(BaseModel): update_comment=update_comment, is_package=is_package ), - output_uri=self._get_base_model().upload_storage_uri or self._default_output_uri + output_uri=self._get_base_model().upload_storage_uri or self._default_output_uri, + id=self.id ) ) return weights_filename_offline or register_uri diff --git a/clearml/task.py b/clearml/task.py index 08fb3a4e..cf5e89cf 100644 --- a/clearml/task.py +++ b/clearml/task.py @@ -3336,10 +3336,24 @@ class Task(_Task): if output_model.get("output_uri"): model.set_upload_destination(output_model.get("output_uri")) model.update_weights(auto_delete_file=False, **output_model["weights"]) + Metrics.report_offline_session( + model, + session_folder, + iteration_offset=iteration_offset, + remote_url=task_holding_reports._get_default_report_storage_uri(), + only_with_id=output_model["id"], + session=task_holding_reports.session + ) # logs TaskHandler.report_offline_session(task_holding_reports, session_folder, iteration_offset=iteration_offset) # metrics - Metrics.report_offline_session(task_holding_reports, session_folder, iteration_offset=iteration_offset) + Metrics.report_offline_session( + task_holding_reports, + session_folder, + iteration_offset=iteration_offset, + only_with_id=export_data["id"], + session=task_holding_reports.session, + ) # print imported results page print('ClearML results page: {}'.format(task_holding_reports.get_output_log_web_page())) task_holding_reports.mark_completed()