ClearML/clearml-server
1
0
Fork 0
mirror of https://github.com/clearml/clearml-server synced 2025-06-26 23:15:47 +00:00
Code Issues Actions Packages Projects Releases Wiki Activity

Fix pre-populate to convert model metadata from the old format

Browse Source
This commit is contained in:
allegroai 2022-03-15 16:30:14 +02:00
parent c04e2e498b
commit 455b5d6758
1 changed files with 22 additions and 2 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

24
apiserver/mongo/initialize/pre_populate.py
View File

@ -21,6 +21,7 @@ from typing import (
Union, Union,
Mapping, Mapping,
IO, IO,
Callable,
) )
from urllib.parse import unquote, urlparse from urllib.parse import unquote, urlparse
from zipfile import ZipFile, ZIP_BZIP2 from zipfile import ZipFile, ZIP_BZIP2
@ -54,6 +55,7 @@ from apiserver.database.model.task.task import (
from apiserver.database.utils import get_options from apiserver.database.utils import get_options
from apiserver.utilities import json from apiserver.utilities import json
from apiserver.utilities.dicts import nested_get, nested_set, nested_delete from apiserver.utilities.dicts import nested_get, nested_set, nested_delete
from apiserver.utilities.parameter_key_escaper import ParameterKeyEscaper
class PrePopulate: class PrePopulate:
@ -744,6 +746,19 @@ class PrePopulate:
module = importlib.import_module(module_name) module = importlib.import_module(module_name)
return getattr(module, class_name) return getattr(module, class_name)
@staticmethod
def _upgrade_model_data(model_data: dict) -> dict:
metadata_key = "metadata"
metadata = model_data.get(metadata_key)
if isinstance(metadata, list):
metadata = {
ParameterKeyEscaper.escape(item["key"]): item
for item in metadata
if isinstance(item, dict) and "key" in item
}
model_data[metadata_key] = metadata
return model_data
@staticmethod @staticmethod
def _upgrade_task_data(task_data: dict) -> dict: def _upgrade_task_data(task_data: dict) -> dict:
""" """
@ -828,9 +843,14 @@ class PrePopulate:
print(f"Writing {cls_.__name__.lower()}s into database") print(f"Writing {cls_.__name__.lower()}s into database")
tasks = [] tasks = []
override_project_count = 0 override_project_count = 0
data_upgrade_funcs: Mapping[Type, Callable] = {
cls.task_cls: cls._upgrade_task_data,
cls.model_cls: cls._upgrade_model_data,
}
for item in cls.json_lines(f): for item in cls.json_lines(f):
if cls_ == cls.task_cls: upgrade_func = data_upgrade_funcs.get(cls_)
item = json.dumps(cls._upgrade_task_data(task_data=json.loads(item))) if upgrade_func:
item = json.dumps(upgrade_func(json.loads(item)))
doc = cls_.from_json(item, created=True) doc = cls_.from_json(item, created=True)
if hasattr(doc, "user"): if hasattr(doc, "user"):