ClearML/clearml
1
0
Fork 0
mirror of https://github.com/clearml/clearml synced 2025-05-11 08:00:51 +00:00
Code Issues Actions 1 Packages Projects Releases Wiki Activity

Fix adding dataset folder with modified files will upload all files instead of just the modified ones

Browse Source
This commit is contained in:
clearml 2025-03-09 18:43:00 +02:00
parent 330abbf9c0
commit aa2f7b0e3e
1 changed files with 16 additions and 0 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

16
clearml/datasets/dataset.py
View File

@ -694,6 +694,8 @@ class Dataset(object):
chunk_size = int(self._dataset_chunk_size_mb if not chunk_size else chunk_size) chunk_size = int(self._dataset_chunk_size_mb if not chunk_size else chunk_size)
upload_futures = [] upload_futures = []
self._fix_dataset_files_parents()
with ThreadPoolExecutor(max_workers=max_workers) as pool: with ThreadPoolExecutor(max_workers=max_workers) as pool:
parallel_zipper = ParallelZipper( parallel_zipper = ParallelZipper(
chunk_size, chunk_size,
@ -1341,6 +1343,20 @@ class Dataset(object):
cls._set_project_system_tags(instance._task) cls._set_project_system_tags(instance._task)
return instance return instance
def _fix_dataset_files_parents(self):
# type: () -> ()
"""
Needed when someone removes and adds the same file -> parent data will be lost
"""
datasets = self._dependency_graph[self._id]
for ds_id in datasets:
dataset = self.get(dataset_id=ds_id)
for parent_file_key, parent_file_value in dataset._dataset_file_entries.items():
if parent_file_key not in self._dataset_file_entries:
continue
if parent_file_value.hash == self._dataset_file_entries[parent_file_key].hash:
self._dataset_file_entries[parent_file_key].parent_dataset_id = ds_id
def _get_total_size_compressed_parents(self): def _get_total_size_compressed_parents(self):
# type: () -> int # type: () -> int
""" """