Fix metrics reporting with OutputModel while in offline mode results in an error (#1172)

clearml/backend_interface/metrics/interface.py

@@ -222,7 +222,13 @@ class Metrics(InterfaceBase):
             if batched_requests:
                 if self._offline_mode:
                     with open(self._offline_log_filename.as_posix(), 'at') as f:
-                        f.write(json.dumps([b.to_dict() for b in batched_requests])+'\n')
+                        requests = []
+                        for b in batched_requests:
+                            request = b.to_dict()
+                            if self._for_model:
+                                request["model_event"] = True
+                            requests.append(request)
+                        f.write(json.dumps(requests)+'\n')
                     return
 
                 req = api_events.AddBatchRequest(requests=batched_requests)
@@ -262,13 +268,16 @@ class Metrics(InterfaceBase):
                 pass
 
     @classmethod
-    def report_offline_session(cls, task, folder, iteration_offset=0):
+    def report_offline_session(cls, task, folder, iteration_offset=0, remote_url=None, only_with_id=None, session=None):
         from ... import StorageManager
         filename = Path(folder) / cls.__offline_filename
         if not filename.is_file():
             return False
-        # noinspection PyProtectedMember
+        if not remote_url:
-        remote_url = task._get_default_report_storage_uri()
+            # noinspection PyProtectedMember
+            remote_url = task._get_default_report_storage_uri()
+        if not session:
+            session = task.session
         if remote_url and remote_url.endswith('/'):
             remote_url = remote_url[:-1]
         uploaded_files = set()
@@ -281,6 +290,8 @@ class Metrics(InterfaceBase):
                     if not line:
                         break
                     list_requests = json.loads(line)
+                    if only_with_id:
+                        list_requests = [r for r in list_requests if r["task"] == only_with_id]
                     for r in list_requests:
                         # noinspection PyBroadException
                         try:
@@ -327,7 +338,7 @@ class Metrics(InterfaceBase):
                     warning('Failed reporting metric, line {} [{}]'.format(i, ex))
                 batch_requests = api_events.AddBatchRequest(requests=list_requests)
                 if batch_requests.requests:
-                    res = task.session.send(batch_requests)
+                    res = session.send(batch_requests)
                     if res and not res.ok():
                         warning("failed logging metric task to backend ({:d} lines, {})".format(
                             len(batch_requests.requests), str(res.meta)))

clearml/model.py

@@ -3,6 +3,7 @@ import os
 import zipfile
 import shutil
 from tempfile import mkstemp
+from uuid import uuid4
 
 import six
 import math
@@ -1070,6 +1071,10 @@ class BaseModel(object):
         except Exception:
             pass
 
+    def get_offline_mode_folder(self):
+        from clearml import Task as OfflineTask
+        return OfflineTask.current_task().get_offline_mode_folder()
+
     def _init_reporter(self):
         if self._reporter:
             return
@@ -2154,6 +2159,16 @@ class OutputModel(BaseModel):
         """
         return self._get_base_model().upload_storage_uri
 
+    @property
+    def id(self):
+        # type: () -> str
+        from clearml import Task as OfflineTask
+        if OfflineTask.is_offline():
+            if not self._base_model_id:
+                self._base_model_id = "offline-{}".format(str(uuid4()).replace("-", ""))
+            return self._base_model_id
+        return super(OutputModel, self).id
+
     def __init__(
         self,
         task=None,  # type: Optional[Task]
@@ -2810,7 +2825,8 @@ class OutputModel(BaseModel):
                     update_comment=update_comment,
                     is_package=is_package
                 ),
-                output_uri=self._get_base_model().upload_storage_uri or self._default_output_uri
+                output_uri=self._get_base_model().upload_storage_uri or self._default_output_uri,
+                id=self.id
             )
         )
         return weights_filename_offline or register_uri

clearml/task.py

@@ -3336,10 +3336,24 @@ class Task(_Task):
             if output_model.get("output_uri"):
                 model.set_upload_destination(output_model.get("output_uri"))
             model.update_weights(auto_delete_file=False, **output_model["weights"])
+            Metrics.report_offline_session(
+                model,
+                session_folder,
+                iteration_offset=iteration_offset,
+                remote_url=task_holding_reports._get_default_report_storage_uri(),
+                only_with_id=output_model["id"],
+                session=task_holding_reports.session
+            )
         # logs
         TaskHandler.report_offline_session(task_holding_reports, session_folder, iteration_offset=iteration_offset)
         # metrics
-        Metrics.report_offline_session(task_holding_reports, session_folder, iteration_offset=iteration_offset)
+        Metrics.report_offline_session(
+            task_holding_reports,
+            session_folder,
+            iteration_offset=iteration_offset,
+            only_with_id=export_data["id"],
+            session=task_holding_reports.session,
+        )
         # print imported results page
         print('ClearML results page: {}'.format(task_holding_reports.get_output_log_web_page()))
         task_holding_reports.mark_completed()