ClearML/clearml
1
0
Fork 0
mirror of https://github.com/clearml/clearml synced 2025-03-03 10:42:00 +00:00
Code Issues Actions Packages Projects Releases Wiki Activity

Fix StorageManager.get_file_size_bytes() and StorageManager.get_metadata() might raise an error even with silence_errors=True

Browse Source
This commit is contained in:
allegroai 2023-02-14 12:25:59 +02:00
parent 7de0644eef
commit 311e05d938
2 changed files with 21 additions and 10 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

29
clearml/storage/helper.py
View File

@ -604,15 +604,17 @@ class StorageHelper(object):
None if the file could not be found or an error occurred. None if the file could not be found or an error occurred.
""" """
obj = self.get_object(remote_url, silence_errors=silence_errors) obj = self.get_object(remote_url, silence_errors=silence_errors)
return self._get_object_size_bytes(obj) return self._get_object_size_bytes(obj, silence_errors)
def _get_object_size_bytes(self, obj): def _get_object_size_bytes(self, obj, silence_errors=False):
# type: (object) -> [int, None] # type: (object) -> [int, None]
""" """
Auxiliary function for `get_object_size_bytes`. Auxiliary function for `get_object_size_bytes`.
Get size of the remote object in bytes. Get size of the remote object in bytes.
:param object obj: The remote object :param object obj: The remote object
:param bool silence_errors: Silence errors that might occur
when fetching the size of the file. Default: False
:return: The size of the object in bytes. :return: The size of the object in bytes.
None if an error occurred. None if an error occurred.
@ -628,17 +630,27 @@ class StorageHelper(object):
size = obj.size size = obj.size
# Google storage has the option to reload the object to get the size # Google storage has the option to reload the object to get the size
if size is None and hasattr(obj, "reload"): if size is None and hasattr(obj, "reload"):
obj.reload() # noinspection PyBroadException
size = obj.size try:
# To catch google.api_core exceptions
obj.reload()
size = obj.size
except Exception as e:
if not silence_errors:
self.log.warning("Failed obtaining object size on reload: {}('{}')".format(
e.__class__.__name__, str(e)))
elif hasattr(obj, "content_length"): elif hasattr(obj, "content_length"):
# noinspection PyBroadException # noinspection PyBroadException
try: try:
# To catch botocore exceptions # To catch botocore exceptions
size = obj.content_length # noqa size = obj.content_length # noqa
except Exception: except Exception as e:
pass if not silence_errors:
except (ValueError, AttributeError, KeyError): self.log.warning("Failed obtaining content_length while getting object size: {}('{}')".format(
pass e.__class__.__name__, str(e)))
except Exception as e:
if not silence_errors:
self.log.warning("Failed getting object size: {}('{}')".format(e.__class__.__name__, str(e)))
return size return size
def get_object_metadata(self, obj): def get_object_metadata(self, obj):
@ -1627,7 +1639,6 @@ class _Boto3Driver(_Driver):
def upload_object_via_stream(self, iterator, container, object_name, callback=None, extra=None, **kwargs): def upload_object_via_stream(self, iterator, container, object_name, callback=None, extra=None, **kwargs):
import boto3.s3.transfer import boto3.s3.transfer
stream = _Stream(iterator) stream = _Stream(iterator)
extra_args = {} extra_args = {}
try: try:

2
clearml/storage/manager.py
View File

@ -341,7 +341,7 @@ class StorageManager(object):
None if the file could not be found or an error occurred. None if the file could not be found or an error occurred.
""" """
helper = StorageHelper.get(remote_url) helper = StorageHelper.get(remote_url)
return helper.get_object_size_bytes(remote_url) return helper.get_object_size_bytes(remote_url, silence_errors)
@classmethod @classmethod
def download_folder( def download_folder(