ClearML/clearml-server
1
0
Fork 0
mirror of https://github.com/clearml/clearml-server synced 2025-02-07 05:27:23 +00:00
Code Issues Actions Packages Projects Releases Wiki Activity

Data tool can now export project trees not starting from the root

Browse Source
This commit is contained in:
clearml 2024-12-05 19:06:56 +02:00
parent 7b5679fd70
commit 8f942f0da2
1 changed files with 30 additions and 7 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

37
apiserver/mongo/initialize/pre_populate.py
View File

@ -28,6 +28,7 @@ from urllib.parse import unquote, urlparse
from uuid import uuid4, UUID, uuid5 from uuid import uuid4, UUID, uuid5
from zipfile import ZipFile, ZIP_BZIP2 from zipfile import ZipFile, ZIP_BZIP2
import attr
import mongoengine import mongoengine
from boltons.iterutils import chunked_iter, first from boltons.iterutils import chunked_iter, first
from furl import furl from furl import furl
@ -84,6 +85,11 @@ class PrePopulate:
user_cls: Type[User] user_cls: Type[User]
auth_user_cls: Type[AuthUser] auth_user_cls: Type[AuthUser]
@attr.s(auto_attribs=True)
class ParentPrefix:
prefix: str
path: Sequence[str]
# noinspection PyTypeChecker # noinspection PyTypeChecker
@classmethod @classmethod
def _init_entity_types(cls): def _init_entity_types(cls):
@ -469,20 +475,35 @@ class PrePopulate:
@classmethod @classmethod
def _check_projects_hierarchy(cls, projects: Set[Project]): def _check_projects_hierarchy(cls, projects: Set[Project]):
""" """
For any exported project all its parents up to the root should be present For the projects that are exported not from the root
fix their parents tree to exclude the not exported parents
""" """
if not projects: if not projects:
return return
project_ids = {p.id for p in projects} project_ids = {p.id for p in projects}
orphans = [p.id for p in projects if p.parent and p.parent not in project_ids] orphans = [p for p in projects if p.parent and p.parent not in project_ids]
if not orphans: if not orphans:
return return
print( prefixes = [
f"ERROR: the following projects are exported without their parents: {orphans}" cls.ParentPrefix(prefix=f"{project.name.rpartition('/')[0]}/", path=project.path)
) for project in orphans
exit(1) ]
prefixes.sort(key=lambda p: len(p.path), reverse=True)
for project in projects:
prefix = first(pref for pref in prefixes if project.path[:len(pref.path)] == pref.path)
if not prefix:
continue
project.path = project.path[len(prefix.path):]
if not project.path:
project.parent = None
project.name = project.name.removeprefix(prefix.prefix)
# print(
# f"ERROR: the following projects are exported without their parents: {orphans}"
# )
# exit(1)
@classmethod @classmethod
def _resolve_entities( def _resolve_entities(
@ -491,6 +512,7 @@ class PrePopulate:
projects: Sequence[str] = None, projects: Sequence[str] = None,
task_statuses: Sequence[str] = None, task_statuses: Sequence[str] = None,
) -> Dict[Type[mongoengine.Document], Set[mongoengine.Document]]: ) -> Dict[Type[mongoengine.Document], Set[mongoengine.Document]]:
# noinspection PyTypeChecker
entities: Dict[Any] = defaultdict(set) entities: Dict[Any] = defaultdict(set)
if projects: if projects:
@ -539,6 +561,7 @@ class PrePopulate:
print("Reading models...") print("Reading models...")
entities[cls.model_cls] = set(cls.model_cls.objects(id__in=list(model_ids))) entities[cls.model_cls] = set(cls.model_cls.objects(id__in=list(model_ids)))
# noinspection PyTypeChecker
return entities return entities
@classmethod @classmethod
@ -1133,7 +1156,7 @@ class PrePopulate:
if isinstance(doc, cls.task_cls): if isinstance(doc, cls.task_cls):
tasks.append(doc) tasks.append(doc)
cls.event_bll.delete_task_events(company_id, doc.id, allow_locked=True) cls.event_bll.delete_task_events(company_id, doc.id)
if tasks: if tasks:
return tasks return tasks