ClearML/clearml
1
0
Fork 0
mirror of https://github.com/clearml/clearml synced 2025-03-03 18:52:12 +00:00
Code Issues Actions Packages Projects Releases Wiki Activity

Add missing skip_zero_size_check argument in StorageManager.download_foler()

Browse Source 
PEP8
This commit is contained in:
allegroai 2022-02-26 13:38:42 +02:00
parent 76559b6e2d
commit a5b16db95c
7 changed files with 16 additions and 19 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

3
clearml/storage/helper.py
View File

@ -1831,7 +1831,6 @@ class _AzureBlobServiceStorageDriver(_Driver):
def is_legacy(self): def is_legacy(self):
return self.__legacy return self.__legacy
@attrs @attrs
class _Object(object): class _Object(object):
container = attrib() container = attrib()
@ -1950,7 +1949,6 @@ class _AzureBlobServiceStorageDriver(_Driver):
download_done = threading.Event() download_done = threading.Event()
download_done.counter = 0 download_done.counter = 0
def callback_func(current, total): def callback_func(current, total):
if callback: if callback:
chunk = current - download_done.counter chunk = current - download_done.counter
@ -1959,7 +1957,6 @@ class _AzureBlobServiceStorageDriver(_Driver):
if current >= total: if current >= total:
download_done.set() download_done.set()
container = obj.container container = obj.container
container.blob_service.MAX_SINGLE_GET_SIZE = 5 * 1024 * 1024 container.blob_service.MAX_SINGLE_GET_SIZE = 5 * 1024 * 1024
_ = container.get_blob_to_path( _ = container.get_blob_to_path(

5
clearml/storage/manager.py
View File

@ -249,9 +249,9 @@ class StorageManager(object):
@classmethod @classmethod
def download_folder( def download_folder(
cls, remote_url, local_folder=None, match_wildcard=None, overwrite=False cls, remote_url, local_folder=None, match_wildcard=None, overwrite=False, skip_zero_size_check=False
): ):
# type: (str, Optional[str], Optional[str], bool) -> Optional[str] # type: (str, Optional[str], Optional[str], bool, bool) -> Optional[str]
""" """
Download remote folder recursively to the local machine, maintaining the sub folder structure Download remote folder recursively to the local machine, maintaining the sub folder structure
from the remote storage. from the remote storage.
@ -271,6 +271,7 @@ class StorageManager(object):
Example: `*.json` Example: `*.json`
:param bool overwrite: If False, and target files exist do not download. :param bool overwrite: If False, and target files exist do not download.
If True always download the remote files. Default False. If True always download the remote files. Default False.
:param bool skip_zero_size_check: If True no error will be raised for files with zero bytes size.
:return: Target local folder :return: Target local folder
""" """

1
examples/frameworks/keras/requirements.txt
View File

@ -1,3 +1,2 @@
Keras
tensorflow>=2.0 tensorflow>=2.0
clearml clearml

16
examples/frameworks/pytorch/requirements.txt
View File

@ -1,9 +1,9 @@
matplotlib
tensorboardX
tensorboard>=1.14.0
torch >= 1.1.0 ; python_version >= '3.8'
torch <= 1.5.1 ; python_version < '3.8'
torchvision
tqdm
jsonschema==3.2.0 ; python_version <= '3.5'
clearml clearml
jsonschema==3.2.0 ; python_version <= '3.5'
matplotlib
pytorch-ignite
tensorboard>=1.14.0
tensorboardX
torch>=1.1.0
torchvision>=0.3.0
tqdm