Add support for hidden internal queues

apiserver/apimodels/queues.py

@@ -30,6 +30,11 @@ class GetByIdRequest(QueueRequest):
     max_task_entries = IntField()
 
 
+class GetAllRequest(Base):
+    max_task_entries = IntField()
+    search_hidden = BoolField(default=False)
+
+
 class GetNextTaskRequest(QueueRequest):
     queue = StringField(required=True)
     get_task_info = BoolField(default=False)

apiserver/bll/queue/queue_bll.py

@@ -3,6 +3,7 @@ from datetime import datetime
 from typing import Callable, Sequence, Optional, Tuple
 
 from elasticsearch import Elasticsearch
+from mongoengine import Q
 
 from apiserver import database
 from apiserver.es_factory import es_factory
@@ -149,6 +150,7 @@ class QueueBLL(object):
         self,
         company_id: str,
         query_dict: dict,
+        query: Q = None,
         max_task_entries: int = None,
         ret_params: dict = None,
     ) -> Sequence[dict]:
@@ -158,6 +160,7 @@ class QueueBLL(object):
                 company=company_id,
                 parameters=query_dict,
                 query_dict=query_dict,
+                query=query,
                 projection_fields=self._get_task_entries_projection(max_task_entries)
                 if max_task_entries
                 else None,
@@ -168,6 +171,7 @@ class QueueBLL(object):
         self,
         company_id: str,
         query_dict: dict,
+        query: Q = None,
         max_task_entries: int = None,
         ret_params: dict = None,
     ) -> Sequence[dict]:
@@ -179,6 +183,7 @@ class QueueBLL(object):
             res = Queue.get_many_with_join(
                 company=company_id,
                 query_dict=query_dict,
+                query=query,
                 override_projection=projection,
                 projection_fields=self._get_task_entries_projection(max_task_entries)
                 if max_task_entries

apiserver/config/default/services/queues.conf

@@ -2,4 +2,7 @@
     metrics_before_from_date: 3600
     # interval in seconds to update queue metrics. Put 0 to disable
     metrics_refresh_interval_sec: 300
+    # the queues with these tags will not be returned from get_all/get_all_ex unless id or name specified
+    # or search_hidden is set
+    hidden_tags: [k8s-glue]
 }

apiserver/schema/services/queues.conf

@@ -152,6 +152,13 @@ get_all_ex {
             type: integer
         }
     }
+    "999.0": ${get_all_ex."2.20"} {
+        request.properties.search_hidden {
+            description: "If set to 'true' then hidden queues are included in the search results"
+            type: boolean
+            default: false
+        }
+    }
 }
 get_all {
     "2.4" {
@@ -244,6 +251,13 @@ get_all {
             type: integer
         }
     }
+    "999.0": ${get_all."2.20"} {
+        request.properties.search_hidden {
+            description: "If set to 'true' then hidden queues are included in the search results"
+            type: boolean
+            default: false
+        }
+    }
 }
 get_default {
     "2.4" {

apiserver/services/queues.py

@@ -1,3 +1,5 @@
+from mongoengine import Q
+
 from apiserver.apimodels.base import UpdateResponse
 from apiserver.apimodels.queues import (
     GetDefaultResp,
@@ -15,10 +17,12 @@ from apiserver.apimodels.queues import (
     DeleteMetadataRequest,
     GetNextTaskRequest,
     GetByIdRequest,
+    GetAllRequest,
 )
 from apiserver.bll.model import Metadata
 from apiserver.bll.queue import QueueBLL
 from apiserver.bll.workers import WorkerBLL
+from apiserver.config_repo import config
 from apiserver.database.model.task.task import Task
 from apiserver.service_repo import APICall, endpoint
 from apiserver.services.utils import (
@@ -51,16 +55,33 @@ def get_by_id(call: APICall):
     call.result.data_model = GetDefaultResp(id=queue.id, name=queue.name)
 
 
+def _hidden_query(data: dict) -> Q:
+    """
+    1. Add only non-hidden queues search condition (unless specifically specified differently)
+    """
+    hidden_tags = config.get("services.queues.hidden_tags", [])
+    if (
+        not hidden_tags
+        or data.get("search_hidden")
+        or data.get("id")
+        or data.get("name")
+    ):
+        return Q()
+
+    return Q(system_tags__nin=hidden_tags)
+
+
 @endpoint("queues.get_all_ex", min_version="2.4")
-def get_all_ex(call: APICall):
+def get_all_ex(call: APICall, company: str, request: GetAllRequest):
     conform_tag_fields(call, call.data)
     ret_params = {}
 
     Metadata.escape_query_parameters(call)
     queues = queue_bll.get_queue_infos(
-        company_id=call.identity.company,
+        company_id=company,
         query_dict=call.data,
-        max_task_entries=call.data.pop("max_task_entries", None),
+        query=_hidden_query(call.data),
+        max_task_entries=request.max_task_entries,
         ret_params=ret_params,
     )
     conform_output_tags(call, queues)
@@ -69,14 +90,15 @@ def get_all_ex(call: APICall):
 
 
 @endpoint("queues.get_all", min_version="2.4")
-def get_all(call: APICall):
+def get_all(call: APICall, company: str, request: GetAllRequest):
     conform_tag_fields(call, call.data)
     ret_params = {}
     Metadata.escape_query_parameters(call)
     queues = queue_bll.get_all(
-        company_id=call.identity.company,
+        company_id=company,
         query_dict=call.data,
-        max_task_entries=call.data.pop("max_task_entries", None),
+        query=_hidden_query(call.data),
+        max_task_entries=request.max_task_entries,
         ret_params=ret_params,
     )
     conform_output_tags(call, queues)

apiserver/tests/automated/test_queues.py

@@ -21,7 +21,7 @@ class TestQueues(TestService):
 
     def test_queue_metrics(self):
         queue_id = self._temp_queue("TestTempQueue")
-        task1 = self._create_temp_queued_task("temp task 1", queue_id)
+        self._create_temp_queued_task("temp task 1", queue_id)
         time.sleep(1)
         task2 = self._create_temp_queued_task("temp task 2", queue_id)
         self.api.queues.get_next_task(queue=queue_id)
@@ -36,6 +36,27 @@ class TestQueues(TestService):
         )
         self.assertMetricQueues(res["queues"], queue_id)
 
+    def test_hidden_queues(self):
+        hidden_name = "TestHiddenQueue"
+        hidden_queue = self._temp_queue(hidden_name, system_tags=["k8s-glue"])
+        non_hidden_queue = self._temp_queue("TestNonHiddenQueue")
+
+        queues = self.api.queues.get_all_ex().queues
+        ids = {q.id for q in queues}
+        self.assertFalse(hidden_queue in ids)
+        self.assertTrue(non_hidden_queue in ids)
+
+        queues = self.api.queues.get_all_ex(search_hidden=True).queues
+        ids = {q.id for q in queues}
+        self.assertTrue(hidden_queue in ids)
+        self.assertTrue(non_hidden_queue in ids)
+
+        queues = self.api.queues.get_all_ex(name=f"^{hidden_name}$").queues
+        self.assertEqual(hidden_queue, queues[0].id)
+
+        queues = self.api.queues.get_all_ex(id=[hidden_queue]).queues
+        self.assertEqual(hidden_queue, queues[0].id)
+
     def test_reset_task(self):
         queue = self._temp_queue("TestTempQueue")
         task = self._temp_task("TempTask", is_development=True)
@@ -207,8 +228,8 @@ class TestQueues(TestService):
             sorted(queue.workers, key=sort_key), sorted(workers, key=sort_key)
         )
 
-    def _temp_queue(self, queue_name, tags=None):
+    def _temp_queue(self, queue_name, **kwargs):
-        return self.create_temp("queues", name=queue_name, tags=tags)
+        return self.create_temp("queues", name=queue_name, **kwargs)
 
     def _temp_task(self, task_name, is_testing=False, is_development=False):
         task_input = dict(