diff --git a/apiserver/apimodels/queues.py b/apiserver/apimodels/queues.py index 23ebd97..3915379 100644 --- a/apiserver/apimodels/queues.py +++ b/apiserver/apimodels/queues.py @@ -30,6 +30,11 @@ class GetByIdRequest(QueueRequest): max_task_entries = IntField() +class GetAllRequest(Base): + max_task_entries = IntField() + search_hidden = BoolField(default=False) + + class GetNextTaskRequest(QueueRequest): queue = StringField(required=True) get_task_info = BoolField(default=False) diff --git a/apiserver/bll/queue/queue_bll.py b/apiserver/bll/queue/queue_bll.py index 9e425b2..6d39fab 100644 --- a/apiserver/bll/queue/queue_bll.py +++ b/apiserver/bll/queue/queue_bll.py @@ -3,6 +3,7 @@ from datetime import datetime from typing import Callable, Sequence, Optional, Tuple from elasticsearch import Elasticsearch +from mongoengine import Q from apiserver import database from apiserver.es_factory import es_factory @@ -149,6 +150,7 @@ class QueueBLL(object): self, company_id: str, query_dict: dict, + query: Q = None, max_task_entries: int = None, ret_params: dict = None, ) -> Sequence[dict]: @@ -158,6 +160,7 @@ class QueueBLL(object): company=company_id, parameters=query_dict, query_dict=query_dict, + query=query, projection_fields=self._get_task_entries_projection(max_task_entries) if max_task_entries else None, @@ -168,6 +171,7 @@ class QueueBLL(object): self, company_id: str, query_dict: dict, + query: Q = None, max_task_entries: int = None, ret_params: dict = None, ) -> Sequence[dict]: @@ -179,6 +183,7 @@ class QueueBLL(object): res = Queue.get_many_with_join( company=company_id, query_dict=query_dict, + query=query, override_projection=projection, projection_fields=self._get_task_entries_projection(max_task_entries) if max_task_entries diff --git a/apiserver/config/default/services/queues.conf b/apiserver/config/default/services/queues.conf index f2adab2..bd8a124 100644 --- a/apiserver/config/default/services/queues.conf +++ b/apiserver/config/default/services/queues.conf @@ -2,4 +2,7 @@ metrics_before_from_date: 3600 # interval in seconds to update queue metrics. Put 0 to disable metrics_refresh_interval_sec: 300 + # the queues with these tags will not be returned from get_all/get_all_ex unless id or name specified + # or search_hidden is set + hidden_tags: [k8s-glue] } \ No newline at end of file diff --git a/apiserver/schema/services/queues.conf b/apiserver/schema/services/queues.conf index ee61317..e9c1d5c 100644 --- a/apiserver/schema/services/queues.conf +++ b/apiserver/schema/services/queues.conf @@ -152,6 +152,13 @@ get_all_ex { type: integer } } + "999.0": ${get_all_ex."2.20"} { + request.properties.search_hidden { + description: "If set to 'true' then hidden queues are included in the search results" + type: boolean + default: false + } + } } get_all { "2.4" { @@ -244,6 +251,13 @@ get_all { type: integer } } + "999.0": ${get_all."2.20"} { + request.properties.search_hidden { + description: "If set to 'true' then hidden queues are included in the search results" + type: boolean + default: false + } + } } get_default { "2.4" { diff --git a/apiserver/services/queues.py b/apiserver/services/queues.py index c4de32d..69227a3 100644 --- a/apiserver/services/queues.py +++ b/apiserver/services/queues.py @@ -1,3 +1,5 @@ +from mongoengine import Q + from apiserver.apimodels.base import UpdateResponse from apiserver.apimodels.queues import ( GetDefaultResp, @@ -15,10 +17,12 @@ from apiserver.apimodels.queues import ( DeleteMetadataRequest, GetNextTaskRequest, GetByIdRequest, + GetAllRequest, ) from apiserver.bll.model import Metadata from apiserver.bll.queue import QueueBLL from apiserver.bll.workers import WorkerBLL +from apiserver.config_repo import config from apiserver.database.model.task.task import Task from apiserver.service_repo import APICall, endpoint from apiserver.services.utils import ( @@ -51,16 +55,33 @@ def get_by_id(call: APICall): call.result.data_model = GetDefaultResp(id=queue.id, name=queue.name) +def _hidden_query(data: dict) -> Q: + """ + 1. Add only non-hidden queues search condition (unless specifically specified differently) + """ + hidden_tags = config.get("services.queues.hidden_tags", []) + if ( + not hidden_tags + or data.get("search_hidden") + or data.get("id") + or data.get("name") + ): + return Q() + + return Q(system_tags__nin=hidden_tags) + + @endpoint("queues.get_all_ex", min_version="2.4") -def get_all_ex(call: APICall): +def get_all_ex(call: APICall, company: str, request: GetAllRequest): conform_tag_fields(call, call.data) ret_params = {} Metadata.escape_query_parameters(call) queues = queue_bll.get_queue_infos( - company_id=call.identity.company, + company_id=company, query_dict=call.data, - max_task_entries=call.data.pop("max_task_entries", None), + query=_hidden_query(call.data), + max_task_entries=request.max_task_entries, ret_params=ret_params, ) conform_output_tags(call, queues) @@ -69,14 +90,15 @@ def get_all_ex(call: APICall): @endpoint("queues.get_all", min_version="2.4") -def get_all(call: APICall): +def get_all(call: APICall, company: str, request: GetAllRequest): conform_tag_fields(call, call.data) ret_params = {} Metadata.escape_query_parameters(call) queues = queue_bll.get_all( - company_id=call.identity.company, + company_id=company, query_dict=call.data, - max_task_entries=call.data.pop("max_task_entries", None), + query=_hidden_query(call.data), + max_task_entries=request.max_task_entries, ret_params=ret_params, ) conform_output_tags(call, queues) diff --git a/apiserver/tests/automated/test_queues.py b/apiserver/tests/automated/test_queues.py index 85e2d01..1605ad2 100644 --- a/apiserver/tests/automated/test_queues.py +++ b/apiserver/tests/automated/test_queues.py @@ -21,7 +21,7 @@ class TestQueues(TestService): def test_queue_metrics(self): queue_id = self._temp_queue("TestTempQueue") - task1 = self._create_temp_queued_task("temp task 1", queue_id) + self._create_temp_queued_task("temp task 1", queue_id) time.sleep(1) task2 = self._create_temp_queued_task("temp task 2", queue_id) self.api.queues.get_next_task(queue=queue_id) @@ -36,6 +36,27 @@ class TestQueues(TestService): ) self.assertMetricQueues(res["queues"], queue_id) + def test_hidden_queues(self): + hidden_name = "TestHiddenQueue" + hidden_queue = self._temp_queue(hidden_name, system_tags=["k8s-glue"]) + non_hidden_queue = self._temp_queue("TestNonHiddenQueue") + + queues = self.api.queues.get_all_ex().queues + ids = {q.id for q in queues} + self.assertFalse(hidden_queue in ids) + self.assertTrue(non_hidden_queue in ids) + + queues = self.api.queues.get_all_ex(search_hidden=True).queues + ids = {q.id for q in queues} + self.assertTrue(hidden_queue in ids) + self.assertTrue(non_hidden_queue in ids) + + queues = self.api.queues.get_all_ex(name=f"^{hidden_name}$").queues + self.assertEqual(hidden_queue, queues[0].id) + + queues = self.api.queues.get_all_ex(id=[hidden_queue]).queues + self.assertEqual(hidden_queue, queues[0].id) + def test_reset_task(self): queue = self._temp_queue("TestTempQueue") task = self._temp_task("TempTask", is_development=True) @@ -207,8 +228,8 @@ class TestQueues(TestService): sorted(queue.workers, key=sort_key), sorted(workers, key=sort_key) ) - def _temp_queue(self, queue_name, tags=None): - return self.create_temp("queues", name=queue_name, tags=tags) + def _temp_queue(self, queue_name, **kwargs): + return self.create_temp("queues", name=queue_name, **kwargs) def _temp_task(self, task_name, is_testing=False, is_development=False): task_input = dict(