ClearML/clearml
1
0
Fork 0
mirror of https://github.com/clearml/clearml synced 2025-06-15 02:50:36 +00:00
Code Issues Actions 1 Packages Projects Releases Wiki Activity

Fix Dataset add single file

Browse Source 
Fix get project name from parent dataset if not specified
This commit is contained in:
allegroai 2021-01-10 12:56:10 +02:00
parent 918d685ed1
commit dfb39249c4
1 changed files with 21 additions and 4 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

25
clearml/datasets/dataset.py
View File

@ -589,15 +589,16 @@ class Dataset(object):
return [f.relative_path for f in matching_errors if f is not None] return [f.relative_path for f in matching_errors if f is not None]
@classmethod @classmethod
def create(cls, dataset_project, dataset_name, parent_datasets=None): def create(cls, dataset_name, dataset_project=None, parent_datasets=None):
# type: (str, str, Optional[Sequence[Union[str, Dataset]]]) -> Dataset # type: (str, Optional[str], Optional[Sequence[Union[str, Dataset]]]) -> Dataset
""" """
Create a new dataset. Multiple dataset parents are supported. Create a new dataset. Multiple dataset parents are supported.
Merging of parent datasets is done based on the order, Merging of parent datasets is done based on the order,
where each one can override overlapping files in the previous parent where each one can override overlapping files in the previous parent
:param dataset_project: Project containing the dataset
:param dataset_name: Naming the new dataset :param dataset_name: Naming the new dataset
:param dataset_project: Project containing the dataset.
If not specified, infer project name form parent datasets
:param parent_datasets: Expand a parent dataset by adding/removing files :param parent_datasets: Expand a parent dataset by adding/removing files
:return: Newly created Dataset object :return: Newly created Dataset object
""" """
@ -605,6 +606,13 @@ class Dataset(object):
if any(not p.is_final() for p in parent_datasets): if any(not p.is_final() for p in parent_datasets):
raise ValueError("Cannot inherit from a parent that was not finalized/closed") raise ValueError("Cannot inherit from a parent that was not finalized/closed")
# get project name
if not dataset_project:
if not parent_datasets:
raise ValueError("Missing dataset project name. Could not infer project name from parent dataset.")
# get project name from parent dataset
dataset_project = parent_datasets[-1]._task.get_project_name()
# merge datasets according to order # merge datasets according to order
dataset_file_entries = {} dataset_file_entries = {}
dependency_graph = {} dependency_graph = {}
@ -847,7 +855,7 @@ class Dataset(object):
if path.is_file(): if path.is_file():
file_entry = self._calc_file_hash( file_entry = self._calc_file_hash(
FileEntry(local_path=path.absolute().as_posix(), FileEntry(local_path=path.absolute().as_posix(),
relative_path=Path(dataset_path or '.') / path.relative_to(local_base_folder), relative_path=(Path(dataset_path or '.') / path.relative_to(local_base_folder)).as_posix(),
parent_dataset_id=self._id)) parent_dataset_id=self._id))
file_entries = [file_entry] file_entries = [file_entry]
else: else:
@ -1242,3 +1250,12 @@ class Dataset(object):
config_type='read-only', config_type='read-only',
config_text=dataset_details config_text=dataset_details
) )
def is_dirty(self):
# type: () -> bool
"""
Return True if the dataset has pending uploads (i.e. we cannot finalize it)
:return: Return True means dataset has pending uploads, call 'upload' to start an upload process.
"""
return self._dirty