mirror of
https://github.com/clearml/clearml
synced 2025-02-24 13:17:03 +00:00
Fix metrics reporting with OutputModel while in offline mode results in an error (#1172)
This commit is contained in:
parent
6d1ac2db6a
commit
ebcda80af5
clearml
@ -222,7 +222,13 @@ class Metrics(InterfaceBase):
|
||||
if batched_requests:
|
||||
if self._offline_mode:
|
||||
with open(self._offline_log_filename.as_posix(), 'at') as f:
|
||||
f.write(json.dumps([b.to_dict() for b in batched_requests])+'\n')
|
||||
requests = []
|
||||
for b in batched_requests:
|
||||
request = b.to_dict()
|
||||
if self._for_model:
|
||||
request["model_event"] = True
|
||||
requests.append(request)
|
||||
f.write(json.dumps(requests)+'\n')
|
||||
return
|
||||
|
||||
req = api_events.AddBatchRequest(requests=batched_requests)
|
||||
@ -262,13 +268,16 @@ class Metrics(InterfaceBase):
|
||||
pass
|
||||
|
||||
@classmethod
|
||||
def report_offline_session(cls, task, folder, iteration_offset=0):
|
||||
def report_offline_session(cls, task, folder, iteration_offset=0, remote_url=None, only_with_id=None, session=None):
|
||||
from ... import StorageManager
|
||||
filename = Path(folder) / cls.__offline_filename
|
||||
if not filename.is_file():
|
||||
return False
|
||||
# noinspection PyProtectedMember
|
||||
remote_url = task._get_default_report_storage_uri()
|
||||
if not remote_url:
|
||||
# noinspection PyProtectedMember
|
||||
remote_url = task._get_default_report_storage_uri()
|
||||
if not session:
|
||||
session = task.session
|
||||
if remote_url and remote_url.endswith('/'):
|
||||
remote_url = remote_url[:-1]
|
||||
uploaded_files = set()
|
||||
@ -281,6 +290,8 @@ class Metrics(InterfaceBase):
|
||||
if not line:
|
||||
break
|
||||
list_requests = json.loads(line)
|
||||
if only_with_id:
|
||||
list_requests = [r for r in list_requests if r["task"] == only_with_id]
|
||||
for r in list_requests:
|
||||
# noinspection PyBroadException
|
||||
try:
|
||||
@ -327,7 +338,7 @@ class Metrics(InterfaceBase):
|
||||
warning('Failed reporting metric, line {} [{}]'.format(i, ex))
|
||||
batch_requests = api_events.AddBatchRequest(requests=list_requests)
|
||||
if batch_requests.requests:
|
||||
res = task.session.send(batch_requests)
|
||||
res = session.send(batch_requests)
|
||||
if res and not res.ok():
|
||||
warning("failed logging metric task to backend ({:d} lines, {})".format(
|
||||
len(batch_requests.requests), str(res.meta)))
|
||||
|
@ -3,6 +3,7 @@ import os
|
||||
import zipfile
|
||||
import shutil
|
||||
from tempfile import mkstemp
|
||||
from uuid import uuid4
|
||||
|
||||
import six
|
||||
import math
|
||||
@ -1070,6 +1071,10 @@ class BaseModel(object):
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
def get_offline_mode_folder(self):
|
||||
from clearml import Task as OfflineTask
|
||||
return OfflineTask.current_task().get_offline_mode_folder()
|
||||
|
||||
def _init_reporter(self):
|
||||
if self._reporter:
|
||||
return
|
||||
@ -2154,6 +2159,16 @@ class OutputModel(BaseModel):
|
||||
"""
|
||||
return self._get_base_model().upload_storage_uri
|
||||
|
||||
@property
|
||||
def id(self):
|
||||
# type: () -> str
|
||||
from clearml import Task as OfflineTask
|
||||
if OfflineTask.is_offline():
|
||||
if not self._base_model_id:
|
||||
self._base_model_id = "offline-{}".format(str(uuid4()).replace("-", ""))
|
||||
return self._base_model_id
|
||||
return super(OutputModel, self).id
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
task=None, # type: Optional[Task]
|
||||
@ -2810,7 +2825,8 @@ class OutputModel(BaseModel):
|
||||
update_comment=update_comment,
|
||||
is_package=is_package
|
||||
),
|
||||
output_uri=self._get_base_model().upload_storage_uri or self._default_output_uri
|
||||
output_uri=self._get_base_model().upload_storage_uri or self._default_output_uri,
|
||||
id=self.id
|
||||
)
|
||||
)
|
||||
return weights_filename_offline or register_uri
|
||||
|
@ -3336,10 +3336,24 @@ class Task(_Task):
|
||||
if output_model.get("output_uri"):
|
||||
model.set_upload_destination(output_model.get("output_uri"))
|
||||
model.update_weights(auto_delete_file=False, **output_model["weights"])
|
||||
Metrics.report_offline_session(
|
||||
model,
|
||||
session_folder,
|
||||
iteration_offset=iteration_offset,
|
||||
remote_url=task_holding_reports._get_default_report_storage_uri(),
|
||||
only_with_id=output_model["id"],
|
||||
session=task_holding_reports.session
|
||||
)
|
||||
# logs
|
||||
TaskHandler.report_offline_session(task_holding_reports, session_folder, iteration_offset=iteration_offset)
|
||||
# metrics
|
||||
Metrics.report_offline_session(task_holding_reports, session_folder, iteration_offset=iteration_offset)
|
||||
Metrics.report_offline_session(
|
||||
task_holding_reports,
|
||||
session_folder,
|
||||
iteration_offset=iteration_offset,
|
||||
only_with_id=export_data["id"],
|
||||
session=task_holding_reports.session,
|
||||
)
|
||||
# print imported results page
|
||||
print('ClearML results page: {}'.format(task_holding_reports.get_output_log_web_page()))
|
||||
task_holding_reports.mark_completed()
|
||||
|
Loading…
Reference in New Issue
Block a user