ClearML/clearml
1
0
Fork 0
mirror of https://github.com/clearml/clearml synced 2025-01-31 09:07:00 +00:00
Code Issues Actions Packages Projects Releases Wiki Activity

Add retries parameter to StorageManager.upload_folder (#1305)

Browse Source
This commit is contained in:
allegroai 2024-07-31 17:28:25 +03:00
parent 165e8a07f9
commit 386e1874be
1 changed files with 4 additions and 2 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

6
clearml/storage/manager.py
View File

@ -211,8 +211,8 @@ class StorageManager(object):
return Session.get_files_server_host() return Session.get_files_server_host()
@classmethod @classmethod
def upload_folder(cls, local_folder, remote_url, match_wildcard=None): def upload_folder(cls, local_folder, remote_url, match_wildcard=None, retries=None):
# type: (str, str, Optional[str]) -> Optional[str] # type: (str, str, Optional[str], Optional[int]) -> Optional[str]
""" """
Upload local folder recursively to a remote storage, maintaining the sub folder structure Upload local folder recursively to a remote storage, maintaining the sub folder structure
in the remote storage. in the remote storage.
@ -231,6 +231,7 @@ class StorageManager(object):
Example: `*.json` Example: `*.json`
Notice: target file size/date are not checked. Default True, always upload. Notice: target file size/date are not checked. Default True, always upload.
Notice if uploading to http, we will always overwrite the target. Notice if uploading to http, we will always overwrite the target.
:param int retries: Number of retries before failing to upload a file in the folder.
:return: Newly uploaded remote URL or None on error. :return: Newly uploaded remote URL or None on error.
""" """
@ -250,6 +251,7 @@ class StorageManager(object):
pool.apply_async( pool.apply_async(
helper.upload, helper.upload,
args=(str(path), str(path).replace(local_folder, remote_url)), args=(str(path), str(path).replace(local_folder, remote_url)),
kwds={"retries": retries if retries else cls._file_upload_retries}
) )
) )