ClearML/clearml
1
0
Fork 0
mirror of https://github.com/clearml/clearml synced 2025-04-07 22:24:30 +00:00
Code Issues Actions Packages Projects Releases Wiki Activity

Fix artifact upload to only use file stream when not uploading locally stored file. Multi-part upload is not supported on stream upload.

Browse Source
This commit is contained in:
allegroai 2020-08-23 01:06:20 +03:00
parent 9c484d3dae
commit e1bbc03001
2 changed files with 17 additions and 8 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

15
trains/backend_interface/metrics/events.py
View File

@ -290,13 +290,18 @@ class UploadEvent(MetricsEventAdapter):
image.save(output, format=image_format, quality=self._quality)
output.seek(0)
else:
local_file = self._local_image_path
# noinspection PyBroadException
try:
output = open(local_file, 'rb')
except Exception as e:
# something happened to the file, we should skip it
output = pathlib2.Path(self._local_image_path)
if not output.is_file():
output = None
except Exception:
output = None
if output is None:
from ...debugging.log import LoggerRoot
LoggerRoot.get_base_logger().warning(str(e))
LoggerRoot.get_base_logger().warning(
'Skipping upload, could not find object file \'{}\''.format(output.as_posix()))
return None
return self.FileEntry(

10
trains/backend_interface/metrics/interface.py
View File

@ -140,7 +140,7 @@ class Metrics(InterfaceBase):
# if entry has no stream, we won't upload it
entry = None
else:
if not hasattr(entry.stream, 'read'):
if not isinstance(entry.stream, Path) and not hasattr(entry.stream, 'read'):
raise ValueError('Invalid file object %s' % entry.stream)
entry.url = url
ev.update(task=self._task_id, iter_offset=self._task_iteration_offset, **kwargs)
@ -165,7 +165,10 @@ class Metrics(InterfaceBase):
try:
storage = self._get_storage(upload_uri)
url = storage.upload_from_stream(e.stream, e.url, retries=self._file_upload_retries)
if isinstance(e.stream, Path):
url = storage.upload(e.stream.as_posix(), e.url, retries=self._file_upload_retries)
else:
url = storage.upload_from_stream(e.stream, e.url, retries=self._file_upload_retries)
e.event.update(url=url)
except Exception as exp:
log.warning("Failed uploading to {} ({})".format(
@ -174,7 +177,8 @@ class Metrics(InterfaceBase):
))
e.set_exception(exp)
e.stream.close()
if not isinstance(e.stream, Path):
e.stream.close()
if e.delete_local_file:
# noinspection PyBroadException
try: