ClearML/clearml
1
0
Fork 0
mirror of https://github.com/clearml/clearml synced 2025-06-23 01:55:38 +00:00
Code Issues Actions Packages Projects Releases Wiki Activity

Fix dataset download (#713)

Browse Source
This commit is contained in:
Daniil Kirsanov 2022-07-11 11:25:46 +03:00 committed by GitHub
parent 8aaa8e7897
commit 6b56b3cfa2
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
1 changed files with 5 additions and 1 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

6
clearml/datasets/dataset.py
View File

@ -2121,6 +2121,9 @@ class Dataset(object):
extract_archive=False, name=self._id) extract_archive=False, name=self._id)
if not local_zip: if not local_zip:
raise ValueError("Could not download dataset id={} entry={}".format(self._id, data_artifact_name)) raise ValueError("Could not download dataset id={} entry={}".format(self._id, data_artifact_name))
return local_zip
def _extract_part(local_zip):
# noinspection PyProtectedMember # noinspection PyProtectedMember
StorageManager._extract_to_cache( StorageManager._extract_to_cache(
cached_file=local_zip, name=self._id, cached_file=local_zip, name=self._id,
@ -2133,7 +2136,8 @@ class Dataset(object):
with ThreadPoolExecutor(max_workers=max_workers) as pool: with ThreadPoolExecutor(max_workers=max_workers) as pool:
for d in data_artifact_entries: for d in data_artifact_entries:
pool.submit(_download_part, d) local_zip = _download_part(d)
pool.submit(_extract_part, local_zip)
return local_folder return local_folder