ClearML/clearml
1
0
Fork 0
mirror of https://github.com/clearml/clearml synced 2025-06-04 03:47:57 +00:00
Code Issues Actions Packages Projects Releases Wiki Activity

Fix PEP8 issues

Browse Source
This commit is contained in:
allegroai 2023-10-13 22:17:29 +03:00
parent 7783e651a5
commit 040fd671a4
1 changed files with 10 additions and 10 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

16
clearml/datasets/dataset.py
View File

@ -3217,18 +3217,18 @@ class Dataset(object):
def _verify_dataset_folder(self, target_base_folder, part, chunk_selection, max_workers): def _verify_dataset_folder(self, target_base_folder, part, chunk_selection, max_workers):
# type: (Path, Optional[int], Optional[dict], Optional[int]) -> bool # type: (Path, Optional[int], Optional[dict], Optional[int]) -> bool
def __verify_file_or_link(target_base_folder, part, chunk_selection, file_entry): def verify_file_or_link(base_folder, ds_part, ds_chunk_selection, file_entry):
# type: (Path, Optional[int], Optional[dict], DatasetFileEntry) -> Optional[bool] # type: (Path, Optional[int], Optional[dict], FileEntry) -> Optional[bool]
# check if we need the file for the requested dataset part # check if we need the file for the requested dataset part
if part is not None: if ds_part is not None:
f_parts = chunk_selection.get(file_entry.parent_dataset_id, []) f_parts = ds_chunk_selection.get(file_entry.parent_dataset_id, [])
# file is not in requested dataset part, no need to check it. # file is not in requested dataset part, no need to check it.
if self._get_chunk_idx_from_artifact_name(file_entry.artifact_name) not in f_parts: if self._get_chunk_idx_from_artifact_name(file_entry.artifact_name) not in f_parts:
return None return None
# check if the local size and the stored size match (faster than comparing hash) # check if the local size and the stored size match (faster than comparing hash)
if (target_base_folder / file_entry.relative_path).stat().st_size != file_entry.size: if (base_folder / file_entry.relative_path).stat().st_size != file_entry.size:
return False return False
return True return True
@ -3241,15 +3241,15 @@ class Dataset(object):
futures_ = [] futures_ = []
with ThreadPoolExecutor(max_workers=max_workers) as tp: with ThreadPoolExecutor(max_workers=max_workers) as tp:
for f in self._dataset_file_entries.values(): for f in self._dataset_file_entries.values():
future = tp.submit(__verify_file_or_link, target_base_folder, part, chunk_selection, f) future = tp.submit(verify_file_or_link, target_base_folder, part, chunk_selection, f)
futures_.append(future) futures_.append(future)
for f in self._dataset_link_entries.values(): for f in self._dataset_link_entries.values():
# don't check whether link is in dataset part, hence None for part and chunk_selection # don't check whether link is in dataset part, hence None for part and chunk_selection
future = tp.submit(__verify_file_or_link, target_base_folder, None, None, f) future = tp.submit(verify_file_or_link, target_base_folder, None, None, f)
futures_.append(future) futures_.append(future)
verified = all(f.result() != False for f in futures_) verified = all(f.result() is not False for f in futures_)
except Exception: except Exception:
verified = False verified = False