ClearML/clearml-server
1
0
Fork 0
mirror of https://github.com/clearml/clearml-server synced 2025-03-03 10:43:10 +00:00
Code Issues Actions Packages Projects Releases Wiki Activity

Support paging in projects.get_model_metadata_values and get_hyperparam_values endpoints

Browse Source
This commit is contained in:
allegroai 2023-07-26 18:38:11 +03:00
parent 3ad636c468
commit 3f22423c3f
6 changed files with 54 additions and 16 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

12
apiserver/apimodels/projects.py
View File

@ -62,15 +62,19 @@ class ProjectUserNamesRequest(MultiProjectRequest):
entity = ActualEnumField(EntityTypeEnum, default=EntityTypeEnum.task)
class ProjectHyperparamValuesRequest(MultiProjectRequest):
class MultiProjectPagedRequest(MultiProjectRequest):
allow_public = fields.BoolField(default=True)
page = fields.IntField(default=0)
page_size = fields.IntField(default=500)
class ProjectHyperparamValuesRequest(MultiProjectPagedRequest):
section = fields.StringField(required=True)
name = fields.StringField(required=True)
allow_public = fields.BoolField(default=True)
class ProjectModelMetadataValuesRequest(MultiProjectRequest):
class ProjectModelMetadataValuesRequest(MultiProjectPagedRequest):
key = fields.StringField(required=True)
allow_public = fields.BoolField(default=True)
class ProjectChildrenType(Enum):

20
apiserver/bll/project/project_queries.py
View File

@ -140,7 +140,11 @@ class ProjectQueries:
name: str,
include_subprojects: bool,
allow_public: bool = True,
page: int = 0,
page_size: int = 500,
) -> ParamValues:
page = max(0, page)
page_size = max(1, page_size)
company_constraint = self._get_company_constraint(company_id, allow_public)
project_constraint = self._get_project_constraint(
project_ids, include_subprojects
@ -160,7 +164,7 @@ class ProjectQueries:
if not last_updated_task:
return 0, []
redis_key = f"hyperparam_values_{company_id}_{'_'.join(project_ids)}_{section}_{name}_{allow_public}"
redis_key = f"hyperparam_values_{company_id}_{'_'.join(project_ids)}_{section}_{name}_{allow_public}_{page}_{page_size}"
last_update = last_updated_task.last_update or datetime.utcnow()
cached_res = self._get_cached_param_values(
key=redis_key,
@ -172,7 +176,6 @@ class ProjectQueries:
if cached_res:
return cached_res
max_values = config.get("services.tasks.hyperparam_values.max_count", 100)
pipeline = [
{
"$match": {
@ -184,7 +187,8 @@ class ProjectQueries:
{"$project": {"value": f"${key_path}.value"}},
{"$group": {"_id": "$value"}},
{"$sort": {"_id": 1}},
{"$limit": max_values},
{"$skip": page * page_size},
{"$limit": page_size},
{
"$group": {
"_id": 1,
@ -311,7 +315,11 @@ class ProjectQueries:
key: str,
include_subprojects: bool,
allow_public: bool = True,
page: int = 0,
page_size: int = 500,
) -> ParamValues:
page = max(0, page)
page_size = max(1, page_size)
company_constraint = self._get_company_constraint(company_id, allow_public)
project_constraint = self._get_project_constraint(
project_ids, include_subprojects
@ -331,7 +339,7 @@ class ProjectQueries:
if not last_updated_model:
return 0, []
redis_key = f"modelmetadata_values_{company_id}_{'_'.join(project_ids)}_{key}_{allow_public}"
redis_key = f"modelmetadata_values_{company_id}_{'_'.join(project_ids)}_{key}_{allow_public}_{page}_{page_size}"
last_update = last_updated_model.last_update or datetime.utcnow()
cached_res = self._get_cached_param_values(
key=redis_key, last_update=last_update
@ -339,7 +347,6 @@ class ProjectQueries:
if cached_res:
return cached_res
max_values = config.get("services.models.metadata_values.max_count", 100)
pipeline = [
{
"$match": {
@ -351,7 +358,8 @@ class ProjectQueries:
{"$project": {"value": f"${key_path}.value"}},
{"$group": {"_id": "$value"}},
{"$sort": {"_id": 1}},
{"$limit": max_values},
{"$skip": page * page_size},
{"$limit": page_size},
{
"$group": {
"_id": 1,

3
apiserver/config/default/services/models.conf
View File

@ -1,7 +1,4 @@
metadata_values {
# maximal amount of distinct model values to retrieve
max_count: 100
# cache ttl sec
cache_ttl_sec: 86400
}

3
apiserver/config/default/services/tasks.conf
View File

@ -11,9 +11,6 @@ non_responsive_tasks_watchdog {
multi_task_histogram_limit: 100
hyperparam_values {
# maximal amount of distinct hyperparam values to retrieve
max_count: 100
# max allowed outdate time for the cashed result
cache_allowed_outdate_sec: 60

28
apiserver/schema/services/projects.conf
View File

@ -979,6 +979,20 @@ get_hyperparam_values {
}
}
}
"999.0": ${get_hyperparam_values."2.13"} {
request.properties {
page {
description: "Page number"
default: 0
type: integer
}
page_size {
description: "Page size"
default: 500
type: integer
}
}
}
}
get_hyper_parameters {
"2.9" {
@ -1076,6 +1090,20 @@ get_model_metadata_values {
}
}
}
"999.0": ${get_model_metadata_values."2.17"} {
request.properties {
page {
description: "Page number"
default: 0
type: integer
}
page_size {
description: "Page size"
default: 500
type: integer
}
}
}
}
get_model_metadata_keys {
"2.17" {

4
apiserver/services/projects.py
View File

@ -405,6 +405,8 @@ def get_model_metadata_values(
key=request.key,
include_subprojects=request.include_subprojects,
allow_public=request.allow_public,
page=request.page,
page_size=request.page_size,
)
call.result.data = {
"total": total,
@ -449,6 +451,8 @@ def get_hyperparam_values(
name=request.name,
include_subprojects=request.include_subprojects,
allow_public=request.allow_public,
page=request.page,
page_size=request.page_size,
)
call.result.data = {
"total": total,