ClearML/clearml
1
0
Fork 0
mirror of https://github.com/clearml/clearml synced 2025-04-16 21:42:10 +00:00
Code Issues Actions Packages Projects Releases Wiki Activity

Fix artifact upload to only use file stream when not uploading locally stored file. Multi-part upload is not supported on stream upload.

Browse Source
This commit is contained in:
allegroai 2020-08-23 01:06:20 +03:00
parent 9c484d3dae
commit e1bbc03001
2 changed files with 17 additions and 8 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

15
trains/backend_interface/metrics/events.py
View File

@ -290,13 +290,18 @@ class UploadEvent(MetricsEventAdapter):
image.save(output, format=image_format, quality=self._quality) image.save(output, format=image_format, quality=self._quality)
output.seek(0) output.seek(0)
else: else:
local_file = self._local_image_path # noinspection PyBroadException
try: try:
output = open(local_file, 'rb') output = pathlib2.Path(self._local_image_path)
except Exception as e: if not output.is_file():
# something happened to the file, we should skip it output = None
except Exception:
output = None
if output is None:
from ...debugging.log import LoggerRoot from ...debugging.log import LoggerRoot
LoggerRoot.get_base_logger().warning(str(e)) LoggerRoot.get_base_logger().warning(
'Skipping upload, could not find object file \'{}\''.format(output.as_posix()))
return None return None
return self.FileEntry( return self.FileEntry(

6
trains/backend_interface/metrics/interface.py
View File

@ -140,7 +140,7 @@ class Metrics(InterfaceBase):
# if entry has no stream, we won't upload it # if entry has no stream, we won't upload it
entry = None entry = None
else: else:
if not hasattr(entry.stream, 'read'): if not isinstance(entry.stream, Path) and not hasattr(entry.stream, 'read'):
raise ValueError('Invalid file object %s' % entry.stream) raise ValueError('Invalid file object %s' % entry.stream)
entry.url = url entry.url = url
ev.update(task=self._task_id, iter_offset=self._task_iteration_offset, **kwargs) ev.update(task=self._task_id, iter_offset=self._task_iteration_offset, **kwargs)
@ -165,6 +165,9 @@ class Metrics(InterfaceBase):
try: try:
storage = self._get_storage(upload_uri) storage = self._get_storage(upload_uri)
if isinstance(e.stream, Path):
url = storage.upload(e.stream.as_posix(), e.url, retries=self._file_upload_retries)
else:
url = storage.upload_from_stream(e.stream, e.url, retries=self._file_upload_retries) url = storage.upload_from_stream(e.stream, e.url, retries=self._file_upload_retries)
e.event.update(url=url) e.event.update(url=url)
except Exception as exp: except Exception as exp:
@ -174,6 +177,7 @@ class Metrics(InterfaceBase):
)) ))
e.set_exception(exp) e.set_exception(exp)
if not isinstance(e.stream, Path):
e.stream.close() e.stream.close()
if e.delete_local_file: if e.delete_local_file:
# noinspection PyBroadException # noinspection PyBroadException