ClearML/clearml-server
1
0
Fork 0
mirror of https://github.com/clearml/clearml-server synced 2025-04-10 07:45:27 +00:00
Code Issues Actions Packages Projects Releases Wiki Activity

Count own tasks/models for projects

Browse Source
This commit is contained in:
allegroai 2022-07-08 17:35:01 +03:00
parent f8b9d9802e
commit 62d5779bd5
7 changed files with 133 additions and 36 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

55
apiserver/bll/project/project_bll.py
View File

@ -183,6 +183,7 @@ class ProjectBLL:
if new_location != old_location:
raise errors.bad_request.CannotUpdateProjectLocation(name=new_name)
fields["name"] = new_name
fields["basename"] = new_name.split("/")[-1]
fields["last_update"] = datetime.utcnow()
updated = project.update(upsert=False, **fields)
@ -225,6 +226,7 @@ class ProjectBLL:
user=user,
company=company,
name=name,
basename=name.split("/")[-1],
description=description,
tags=tags,
system_tags=system_tags,
@ -328,6 +330,7 @@ class ProjectBLL:
project_ids: Sequence[str],
specific_state: Optional[EntityVisibility] = None,
filter_: Mapping[str, Any] = None,
users: Sequence[str] = None,
) -> Tuple[Sequence, Sequence]:
archived = EntityVisibility.archived.value
@ -352,7 +355,10 @@ class ProjectBLL:
# count tasks per project per status
{
"$match": cls.get_match_conditions(
company=company_id, project_ids=project_ids, filter_=filter_
company=company_id,
project_ids=project_ids,
filter_=filter_,
users=users,
)
},
ensure_valid_fields(),
@ -469,7 +475,10 @@ class ProjectBLL:
{
"$match": {
**cls.get_match_conditions(
company=company_id, project_ids=project_ids, filter_=filter_
company=company_id,
project_ids=project_ids,
filter_=filter_,
users=users,
),
**get_state_filter(),
}
@ -516,13 +525,18 @@ class ProjectBLL:
include_children: bool = True,
search_hidden: bool = False,
filter_: Mapping[str, Any] = None,
users: Sequence[str] = None,
user_active_project_ids: Sequence[str] = None,
) -> Tuple[Dict[str, dict], Dict[str, dict]]:
if not project_ids:
return {}, {}
child_projects = (
_get_sub_projects(
project_ids, _only=("id", "name"), search_hidden=search_hidden
project_ids,
_only=("id", "name"),
search_hidden=search_hidden,
allowed_ids=user_active_project_ids,
)
if include_children
else {}
@ -535,6 +549,7 @@ class ProjectBLL:
project_ids=list(project_ids_with_children),
specific_state=specific_state,
filter_=filter_,
users=users,
)
default_counts = dict.fromkeys(get_options(TaskStatus), 0)
@ -701,7 +716,7 @@ class ProjectBLL:
users: Sequence[str],
project_ids: Optional[Sequence[str]] = None,
allow_public: bool = True,
) -> Sequence[str]:
) -> Tuple[Sequence[str], Sequence[str]]:
"""
Get the projects ids where user created any tasks including all the parents of these projects
If project ids are specified then filter the results by these project ids
@ -725,13 +740,16 @@ class ProjectBLL:
res = list(res)
if not res:
return res
return res, res
ids_with_parents = _ids_with_parents(res)
if project_ids:
return [pid for pid in ids_with_parents if pid in project_ids]
user_active_project_ids = _ids_with_parents(res)
filtered_ids = (
list(set(user_active_project_ids) & set(project_ids))
if project_ids
else list(user_active_project_ids)
)
return ids_with_parents
return filtered_ids, user_active_project_ids
@classmethod
def get_task_parents(
@ -800,12 +818,18 @@ class ProjectBLL:
@staticmethod
def get_match_conditions(
company: str, project_ids: Sequence[str], filter_: Mapping[str, Any]
company: str,
project_ids: Sequence[str],
filter_: Mapping[str, Any],
users: Sequence[str],
):
conditions = {
"company": {"$in": [None, "", company]},
"project": {"$in": project_ids},
}
if users:
conditions["user"] = {"$in": users}
if not filter_:
return conditions
@ -828,7 +852,11 @@ class ProjectBLL:
@classmethod
def calc_own_contents(
cls, company: str, project_ids: Sequence[str], filter_: Mapping[str, Any] = None
cls,
company: str,
project_ids: Sequence[str],
filter_: Mapping[str, Any] = None,
users: Sequence[str] = None,
) -> Dict[str, dict]:
"""
Returns the amount of task/models per requested project
@ -841,7 +869,10 @@ class ProjectBLL:
pipeline = [
{
"$match": cls.get_match_conditions(
company=company, project_ids=project_ids, filter_=filter_
company=company,
project_ids=project_ids,
filter_=filter_,
users=users,
)
},
{"$project": {"project": 1}},

5
apiserver/bll/project/sub_projects.py
View File

@ -51,6 +51,7 @@ def _ensure_project(
created=now,
last_update=now,
name=name,
basename=name.split("/")[-1],
**(creation_params or dict(description="")),
)
parent = _ensure_project(company, user, location, creation_params=creation_params)
@ -104,6 +105,7 @@ def _get_sub_projects(
project_ids: Sequence[str],
_only: Sequence[str] = ("id", "path"),
search_hidden=True,
allowed_ids: Sequence[str] = None,
) -> Mapping[str, Sequence[Project]]:
"""
Return the list of child projects of all the levels for the parent project ids
@ -111,6 +113,9 @@ def _get_sub_projects(
query = dict(path__in=project_ids)
if not search_hidden:
query["system_tags__nin"] = [EntityVisibility.hidden.value]
if allowed_ids:
query["id__in"] = allowed_ids
qs = Project.objects(**query)
if _only:
_only = set(_only) | {"path"}

11
apiserver/config/default/services/events.conf
View File

@ -12,12 +12,23 @@ events_retrieval {
# should not exceed the amount of concurrent connections set in the ES driver
max_metrics_concurrency: 4
# If set then max_metrics_count and max_variants_count are calculated dynamically on user data
dynamic_metrics_count: true
# The percentage from the ES aggs limit (10000) to use for the max_metrics and max_variants calculation
dynamic_metrics_count_threshold: 80
# the max amount of metrics to aggregate on
max_metrics_count: 100
# the max amount of variants to aggregate on
max_variants_count: 100
debug_images {
# Allow to return the debug images for the variants with uninitialized valid iterations border
allow_uninitialized_variants: true
}
max_raw_scalars_size: 200000
scroll_id_key: "cTN5VEtWEC6QrHvUl0FTx9kNyO0CcCK1p57akxma"

56
apiserver/database/model/base.py
View File

@ -704,6 +704,45 @@ class GetMixin(PropsMixin):
v for k, v in cls._field_collation_overrides.items() if field.startswith(k)
)
@classmethod
def get_count(
cls: Union["GetMixin", Document],
company,
query_dict: dict = None,
query_options: QueryParameterOptions = None,
query: Q = None,
allow_public=False,
) -> int:
_query = cls._get_combined_query(
company=company,
query_dict=query_dict,
query_options=query_options,
query=query,
allow_public=allow_public,
)
return cls.objects(_query).count()
@classmethod
def _get_combined_query(
cls,
company,
query_dict: dict = None,
query_options: QueryParameterOptions = None,
query: Q = None,
allow_public=False,
) -> Q:
if query_dict is not None:
q = cls.prepare_query(
parameters=query_dict,
company=company,
parameters_options=query_options,
allow_public=allow_public,
)
else:
q = cls._prepare_perm_query(company, allow_public=allow_public)
return (q & query) if query else q
@classmethod
def get_many(
cls,
@ -749,16 +788,13 @@ class GetMixin(PropsMixin):
if override_collation:
break
if query_dict is not None:
q = cls.prepare_query(
parameters=query_dict,
company=company,
parameters_options=query_options,
allow_public=allow_public,
)
else:
q = cls._prepare_perm_query(company, allow_public=allow_public)
_query = (q & query) if query else q
_query = cls._get_combined_query(
company=company,
query_dict=query_dict,
query_options=query_options,
query=query,
allow_public=allow_public,
)
if return_dicts:
data_getter = partial(

4
apiserver/database/model/project.py
View File

@ -9,7 +9,7 @@ from apiserver.database.model.base import GetMixin
class Project(AttributedDocument):
get_all_query_options = GetMixin.QueryParameterOptions(
pattern_fields=("name", "description"),
pattern_fields=("name", "basename", "description"),
list_fields=("tags", "system_tags", "id", "parent", "path"),
range_fields=("last_update",),
)
@ -21,6 +21,7 @@ class Project(AttributedDocument):
"parent",
"path",
("company", "name"),
("company", "basename"),
{
"name": "%s.project.main_text_index" % Database.backend,
"fields": ["$name", "$id", "$description"],
@ -37,6 +38,7 @@ class Project(AttributedDocument):
min_length=3,
sparse=True,
)
basename = StrippedStringField(required=True)
description = StringField()
created = DateTimeField(required=True)
tags = SafeSortedListField(StringField(required=True))

32
apiserver/schema/services/projects.conf
View File

@ -25,6 +25,10 @@ _definitions {
description: "Project name"
type: string
}
basename {
description: "Project base name"
type: string
}
description {
description: "Project description"
type: string
@ -156,6 +160,10 @@ _definitions {
description: "Project name"
type: string
}
basename {
description: "Project base name"
type: string
}
description {
description: "Project description"
type: string
@ -214,6 +222,14 @@ _definitions {
}
}
}
own_tasks {
description: "The amount of tasks under this project (without children projects). Returned if 'check_own_contents' flag is set in the request"
type: integer
}
own_models {
description: "The amount of models under this project (without children projects). Returned if 'check_own_contents' flag is set in the request"
type: integer
}
}
}
metric_variant_result {
@ -385,6 +401,10 @@ get_all {
description: "Get only projects whose name matches this pattern (python regular expression syntax)"
type: string
}
basename {
description: "Project base name"
type: string
}
description {
description: "Get only projects whose description matches this pattern (python regular expression syntax)"
type: string
@ -530,18 +550,6 @@ get_all_ex {
}
}
}
response {
properties {
own_tasks {
description: "The amount of tasks under this project (without children projects). Returned if 'check_own_contents' flag is set in the request"
type: integer
}
own_models {
description: "The amount of models under this project (without children projects). Returned if 'check_own_contents' flag is set in the request"
type: integer
}
}
}
}
"2.14": ${get_all_ex."2.13"} {
request.properties.search_hidden {

6
apiserver/services/projects.py
View File

@ -110,8 +110,9 @@ def get_all_ex(call: APICall, company_id: str, request: ProjectsGetRequest):
data, shallow_search=request.shallow_search,
)
with TimingContext("mongo", "projects_get_all"):
user_active_project_ids = None
if request.active_users:
ids = project_bll.get_projects_with_active_user(
ids, user_active_project_ids = project_bll.get_projects_with_active_user(
company=company_id,
users=request.active_users,
project_ids=requested_ids,
@ -139,6 +140,7 @@ def get_all_ex(call: APICall, company_id: str, request: ProjectsGetRequest):
company=company_id,
project_ids=list(existing_requested_ids),
filter_=request.include_stats_filter,
users=request.active_users,
)
for project in projects:
project.update(**contents.get(project["id"], {}))
@ -156,6 +158,8 @@ def get_all_ex(call: APICall, company_id: str, request: ProjectsGetRequest):
include_children=request.stats_with_children,
search_hidden=request.search_hidden,
filter_=request.include_stats_filter,
users=request.active_users,
user_active_project_ids=user_active_project_ids,
)
for project in projects: