ClearML/clearml
1
0
Fork 0
mirror of https://github.com/clearml/clearml synced 2025-03-03 10:42:00 +00:00
Code Issues Actions Packages Projects Releases Wiki Activity

Fix background upload retries with Google Storage (gs://)

Browse Source
This commit is contained in:
allegroai 2021-05-19 15:26:43 +03:00
parent 91ff64b1f6
commit d3929033c0
1 changed files with 7 additions and 4 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

11
clearml/storage/helper.py
View File

@ -921,7 +921,10 @@ class StorageHelper(object):
last_ex = None last_ex = None
for i in range(max(1, retries)): for i in range(max(1, retries)):
try: try:
self._upload_from_file(local_path=src_path, dest_path=dest_path, extra=extra) if not self._upload_from_file(local_path=src_path, dest_path=dest_path, extra=extra):
# retry if failed
last_ex = ValueError("Upload failed")
continue
last_ex = None last_ex = None
break break
except Exception as e: except Exception as e:
@ -1632,7 +1635,7 @@ class _AzureBlobServiceStorageDriver(_Driver):
return self._containers[container_name] return self._containers[container_name]
def upload_object_via_stream(self, iterator, container, object_name, callback=None, extra=None, **kwargs): def upload_object_via_stream(self, iterator, container, object_name, callback=None, extra=None, **kwargs):
from azure.common import AzureHttpError from azure.common import AzureHttpError # noqa
blob_name = self._blob_name_from_object_path(object_name, container.name) # noqa: F841 blob_name = self._blob_name_from_object_path(object_name, container.name) # noqa: F841
try: try:
@ -1654,12 +1657,12 @@ class _AzureBlobServiceStorageDriver(_Driver):
return False return False
def upload_object(self, file_path, container, object_name, callback=None, extra=None, **kwargs): def upload_object(self, file_path, container, object_name, callback=None, extra=None, **kwargs):
from azure.common import AzureHttpError from azure.common import AzureHttpError # noqa
blob_name = self._blob_name_from_object_path(object_name, container.name) blob_name = self._blob_name_from_object_path(object_name, container.name)
stream = None stream = None
try: try:
from azure.storage.blob import ContentSettings from azure.storage.blob import ContentSettings # noqa
from mimetypes import guess_type from mimetypes import guess_type
container.blob_service.MAX_SINGLE_PUT_SIZE = 16 * 1024 * 1024 container.blob_service.MAX_SINGLE_PUT_SIZE = 16 * 1024 * 1024
container.blob_service.socket_timeout = (300, 2000) container.blob_service.socket_timeout = (300, 2000)