1
0
mirror of https://github.com/clearml/clearml synced 2025-02-24 13:17:03 +00:00

Fix metrics reporting with OutputModel while in offline mode results in an error ()

This commit is contained in:
allegroai 2024-03-24 10:58:37 +02:00
parent 6d1ac2db6a
commit ebcda80af5
3 changed files with 48 additions and 7 deletions
clearml
backend_interface/metrics
model.pytask.py

View File

@ -222,7 +222,13 @@ class Metrics(InterfaceBase):
if batched_requests:
if self._offline_mode:
with open(self._offline_log_filename.as_posix(), 'at') as f:
f.write(json.dumps([b.to_dict() for b in batched_requests])+'\n')
requests = []
for b in batched_requests:
request = b.to_dict()
if self._for_model:
request["model_event"] = True
requests.append(request)
f.write(json.dumps(requests)+'\n')
return
req = api_events.AddBatchRequest(requests=batched_requests)
@ -262,13 +268,16 @@ class Metrics(InterfaceBase):
pass
@classmethod
def report_offline_session(cls, task, folder, iteration_offset=0):
def report_offline_session(cls, task, folder, iteration_offset=0, remote_url=None, only_with_id=None, session=None):
from ... import StorageManager
filename = Path(folder) / cls.__offline_filename
if not filename.is_file():
return False
# noinspection PyProtectedMember
remote_url = task._get_default_report_storage_uri()
if not remote_url:
# noinspection PyProtectedMember
remote_url = task._get_default_report_storage_uri()
if not session:
session = task.session
if remote_url and remote_url.endswith('/'):
remote_url = remote_url[:-1]
uploaded_files = set()
@ -281,6 +290,8 @@ class Metrics(InterfaceBase):
if not line:
break
list_requests = json.loads(line)
if only_with_id:
list_requests = [r for r in list_requests if r["task"] == only_with_id]
for r in list_requests:
# noinspection PyBroadException
try:
@ -327,7 +338,7 @@ class Metrics(InterfaceBase):
warning('Failed reporting metric, line {} [{}]'.format(i, ex))
batch_requests = api_events.AddBatchRequest(requests=list_requests)
if batch_requests.requests:
res = task.session.send(batch_requests)
res = session.send(batch_requests)
if res and not res.ok():
warning("failed logging metric task to backend ({:d} lines, {})".format(
len(batch_requests.requests), str(res.meta)))

View File

@ -3,6 +3,7 @@ import os
import zipfile
import shutil
from tempfile import mkstemp
from uuid import uuid4
import six
import math
@ -1070,6 +1071,10 @@ class BaseModel(object):
except Exception:
pass
def get_offline_mode_folder(self):
from clearml import Task as OfflineTask
return OfflineTask.current_task().get_offline_mode_folder()
def _init_reporter(self):
if self._reporter:
return
@ -2154,6 +2159,16 @@ class OutputModel(BaseModel):
"""
return self._get_base_model().upload_storage_uri
@property
def id(self):
# type: () -> str
from clearml import Task as OfflineTask
if OfflineTask.is_offline():
if not self._base_model_id:
self._base_model_id = "offline-{}".format(str(uuid4()).replace("-", ""))
return self._base_model_id
return super(OutputModel, self).id
def __init__(
self,
task=None, # type: Optional[Task]
@ -2810,7 +2825,8 @@ class OutputModel(BaseModel):
update_comment=update_comment,
is_package=is_package
),
output_uri=self._get_base_model().upload_storage_uri or self._default_output_uri
output_uri=self._get_base_model().upload_storage_uri or self._default_output_uri,
id=self.id
)
)
return weights_filename_offline or register_uri

View File

@ -3336,10 +3336,24 @@ class Task(_Task):
if output_model.get("output_uri"):
model.set_upload_destination(output_model.get("output_uri"))
model.update_weights(auto_delete_file=False, **output_model["weights"])
Metrics.report_offline_session(
model,
session_folder,
iteration_offset=iteration_offset,
remote_url=task_holding_reports._get_default_report_storage_uri(),
only_with_id=output_model["id"],
session=task_holding_reports.session
)
# logs
TaskHandler.report_offline_session(task_holding_reports, session_folder, iteration_offset=iteration_offset)
# metrics
Metrics.report_offline_session(task_holding_reports, session_folder, iteration_offset=iteration_offset)
Metrics.report_offline_session(
task_holding_reports,
session_folder,
iteration_offset=iteration_offset,
only_with_id=export_data["id"],
session=task_holding_reports.session,
)
# print imported results page
print('ClearML results page: {}'.format(task_holding_reports.get_output_log_web_page()))
task_holding_reports.mark_completed()