diff --git a/apiserver/apimodels/projects.py b/apiserver/apimodels/projects.py index eb92412..2e72c9e 100644 --- a/apiserver/apimodels/projects.py +++ b/apiserver/apimodels/projects.py @@ -62,15 +62,19 @@ class ProjectUserNamesRequest(MultiProjectRequest): entity = ActualEnumField(EntityTypeEnum, default=EntityTypeEnum.task) -class ProjectHyperparamValuesRequest(MultiProjectRequest): +class MultiProjectPagedRequest(MultiProjectRequest): + allow_public = fields.BoolField(default=True) + page = fields.IntField(default=0) + page_size = fields.IntField(default=500) + + +class ProjectHyperparamValuesRequest(MultiProjectPagedRequest): section = fields.StringField(required=True) name = fields.StringField(required=True) - allow_public = fields.BoolField(default=True) -class ProjectModelMetadataValuesRequest(MultiProjectRequest): +class ProjectModelMetadataValuesRequest(MultiProjectPagedRequest): key = fields.StringField(required=True) - allow_public = fields.BoolField(default=True) class ProjectChildrenType(Enum): diff --git a/apiserver/bll/project/project_queries.py b/apiserver/bll/project/project_queries.py index 76a2af4..6e9619a 100644 --- a/apiserver/bll/project/project_queries.py +++ b/apiserver/bll/project/project_queries.py @@ -140,7 +140,11 @@ class ProjectQueries: name: str, include_subprojects: bool, allow_public: bool = True, + page: int = 0, + page_size: int = 500, ) -> ParamValues: + page = max(0, page) + page_size = max(1, page_size) company_constraint = self._get_company_constraint(company_id, allow_public) project_constraint = self._get_project_constraint( project_ids, include_subprojects @@ -160,7 +164,7 @@ class ProjectQueries: if not last_updated_task: return 0, [] - redis_key = f"hyperparam_values_{company_id}_{'_'.join(project_ids)}_{section}_{name}_{allow_public}" + redis_key = f"hyperparam_values_{company_id}_{'_'.join(project_ids)}_{section}_{name}_{allow_public}_{page}_{page_size}" last_update = last_updated_task.last_update or datetime.utcnow() cached_res = self._get_cached_param_values( key=redis_key, @@ -172,7 +176,6 @@ class ProjectQueries: if cached_res: return cached_res - max_values = config.get("services.tasks.hyperparam_values.max_count", 100) pipeline = [ { "$match": { @@ -184,7 +187,8 @@ class ProjectQueries: {"$project": {"value": f"${key_path}.value"}}, {"$group": {"_id": "$value"}}, {"$sort": {"_id": 1}}, - {"$limit": max_values}, + {"$skip": page * page_size}, + {"$limit": page_size}, { "$group": { "_id": 1, @@ -311,7 +315,11 @@ class ProjectQueries: key: str, include_subprojects: bool, allow_public: bool = True, + page: int = 0, + page_size: int = 500, ) -> ParamValues: + page = max(0, page) + page_size = max(1, page_size) company_constraint = self._get_company_constraint(company_id, allow_public) project_constraint = self._get_project_constraint( project_ids, include_subprojects @@ -331,7 +339,7 @@ class ProjectQueries: if not last_updated_model: return 0, [] - redis_key = f"modelmetadata_values_{company_id}_{'_'.join(project_ids)}_{key}_{allow_public}" + redis_key = f"modelmetadata_values_{company_id}_{'_'.join(project_ids)}_{key}_{allow_public}_{page}_{page_size}" last_update = last_updated_model.last_update or datetime.utcnow() cached_res = self._get_cached_param_values( key=redis_key, last_update=last_update @@ -339,7 +347,6 @@ class ProjectQueries: if cached_res: return cached_res - max_values = config.get("services.models.metadata_values.max_count", 100) pipeline = [ { "$match": { @@ -351,7 +358,8 @@ class ProjectQueries: {"$project": {"value": f"${key_path}.value"}}, {"$group": {"_id": "$value"}}, {"$sort": {"_id": 1}}, - {"$limit": max_values}, + {"$skip": page * page_size}, + {"$limit": page_size}, { "$group": { "_id": 1, diff --git a/apiserver/config/default/services/models.conf b/apiserver/config/default/services/models.conf index a637440..5ee5741 100644 --- a/apiserver/config/default/services/models.conf +++ b/apiserver/config/default/services/models.conf @@ -1,7 +1,4 @@ metadata_values { - # maximal amount of distinct model values to retrieve - max_count: 100 - # cache ttl sec cache_ttl_sec: 86400 } diff --git a/apiserver/config/default/services/tasks.conf b/apiserver/config/default/services/tasks.conf index c51cccb..af7c76d 100644 --- a/apiserver/config/default/services/tasks.conf +++ b/apiserver/config/default/services/tasks.conf @@ -11,9 +11,6 @@ non_responsive_tasks_watchdog { multi_task_histogram_limit: 100 hyperparam_values { - # maximal amount of distinct hyperparam values to retrieve - max_count: 100 - # max allowed outdate time for the cashed result cache_allowed_outdate_sec: 60 diff --git a/apiserver/schema/services/projects.conf b/apiserver/schema/services/projects.conf index 0bbd5d4..5c98b87 100644 --- a/apiserver/schema/services/projects.conf +++ b/apiserver/schema/services/projects.conf @@ -979,6 +979,20 @@ get_hyperparam_values { } } } + "999.0": ${get_hyperparam_values."2.13"} { + request.properties { + page { + description: "Page number" + default: 0 + type: integer + } + page_size { + description: "Page size" + default: 500 + type: integer + } + } + } } get_hyper_parameters { "2.9" { @@ -1076,6 +1090,20 @@ get_model_metadata_values { } } } + "999.0": ${get_model_metadata_values."2.17"} { + request.properties { + page { + description: "Page number" + default: 0 + type: integer + } + page_size { + description: "Page size" + default: 500 + type: integer + } + } + } } get_model_metadata_keys { "2.17" { diff --git a/apiserver/services/projects.py b/apiserver/services/projects.py index 4d74138..659e38f 100644 --- a/apiserver/services/projects.py +++ b/apiserver/services/projects.py @@ -405,6 +405,8 @@ def get_model_metadata_values( key=request.key, include_subprojects=request.include_subprojects, allow_public=request.allow_public, + page=request.page, + page_size=request.page_size, ) call.result.data = { "total": total, @@ -449,6 +451,8 @@ def get_hyperparam_values( name=request.name, include_subprojects=request.include_subprojects, allow_public=request.allow_public, + page=request.page, + page_size=request.page_size, ) call.result.data = { "total": total,