ClearML/clearml-server
1
0
Fork 0
mirror of https://github.com/clearml/clearml-server synced 2025-06-26 23:15:47 +00:00
Code Issues Actions Packages Projects Releases Wiki Activity

Optimize MongoDB indices usage for large dbs

Browse Source
This commit is contained in:
clearml
2024-12-05 22:17:13 +02:00
parent a1956cdd83
commit ee9f45ea61
11 changed files with 47 additions and 19 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
apiserver/database/model/base.py
View File

@@ -1394,7 +1394,7 @@ class DbModelMixin(GetMixin, ProperDictMixin, UpdateMixin):
else:
items = list(
cls.objects(
id__in=ids, company__in=(None, ""), company_origin=company_id
id__in=ids, company="", company_origin=company_id
).only("id")
)
update: dict = dict(set__company=company_id, unset__company_origin=1)

12
apiserver/database/model/model.py
View File

@@ -37,10 +37,18 @@ class Model(AttributedDocument):
"project",
"task",
"last_update",
("company", "framework"),
("company", "last_update"),
("company", "name"),
("company", "user"),
("company", "uri"),
# distinct queries support
("company", "tags"),
("company", "system_tags"),
("company", "project", "tags"),
("company", "project", "system_tags"),
("company", "user"),
("company", "project", "user"),
("company", "framework"),
("company", "project", "framework"),
{
"name": "%s.model.main_text_index" % Database.backend,
"fields": ["$name", "$id", "$comment", "$parent", "$task", "$project"],

14
apiserver/database/model/task/task.py
View File

@@ -183,9 +183,8 @@ class Task(AttributedDocument):
"status_changed",
"models.input.model",
("company", "name"),
("company", "user"),
("company", "status", "type"),
("company", "system_tags", "last_update"),
("company", "last_update", "system_tags"),
("company", "type", "system_tags", "status"),
("company", "project", "type", "system_tags", "status"),
("status", "last_update"), # for maintenance tasks
@@ -193,6 +192,17 @@ class Task(AttributedDocument):
"fields": ["company", "project"],
"collation": AttributedDocument._numeric_locale,
},
# distinct queries support
("company", "tags"),
("company", "system_tags"),
("company", "project", "tags"),
("company", "project", "system_tags"),
("company", "user"),
("company", "project", "user"),
("company", "parent"),
("company", "project", "parent"),
("company", "type"),
("company", "project", "type"),
{
"name": "%s.task.main_text_index" % Database.backend,
"fields": [

4
apiserver/database/utils.py
View File

@@ -121,8 +121,8 @@ def init_cls_from_base(cls, instance):
)
def get_company_or_none_constraint(company=None):
return Q(company__in=(company, None, "")) | Q(company__exists=False)
def get_company_or_none_constraint(company=""):
return Q(company__in=list({company, ""}))
def field_does_not_exist(field: str, empty_value=None, is_list=False) -> Q: