Add organization.get_tags to obtain the set of all used task, model, queue and project tags

server/database/model/base.py

@@ -3,7 +3,7 @@ from collections import namedtuple
 from functools import reduce
 from typing import Collection, Sequence, Union, Optional
 
-from boltons.iterutils import first
+from boltons.iterutils import first, bucketize
 from dateutil.parser import parse as parse_datetime
 from mongoengine import Q, Document, ListField, StringField
 from pymongo.command_cursor import CommandCursor
@@ -98,6 +98,37 @@ class GetMixin(PropsMixin):
             self.list_fields = list_fields
             self.pattern_fields = pattern_fields
 
+    class ListFieldBucketHelper:
+        op_prefix = "__$"
+        legacy_exclude_prefix = "-"
+
+        _default = "in"
+        _ops = {"not": "nin"}
+        _next = _default
+
+        def __init__(self, legacy=False):
+            self._legacy = legacy
+
+        def key(self, v):
+            if v is None:
+                self._next = self._default
+                return self._default
+            elif self._legacy and v.startswith(self.legacy_exclude_prefix):
+                self._next = self._default
+                return self._ops["not"]
+            elif v.startswith(self.op_prefix):
+                self._next = self._ops.get(v[len(self.op_prefix) :], self._default)
+                return None
+
+            next_ = self._next
+            self._next = self._default
+            return next_
+
+        def value_transform(self, v):
+            if self._legacy and v and v.startswith(self.legacy_exclude_prefix):
+                return v[len(self.legacy_exclude_prefix) :]
+            return v
+
     get_all_query_options = QueryParameterOptions()
 
     @classmethod
@@ -175,17 +206,7 @@ class GetMixin(PropsMixin):
             for field in tuple(opts.list_fields or ()):
                 data = parameters.pop(field, None)
                 if data:
-                    if not isinstance(data, (list, tuple)):
-                        raise MakeGetAllQueryError("expected list", field)
-                    exclude = [t for t in data if t.startswith("-")]
-                    include = list(set(data).difference(exclude))
-                    mongoengine_field = field.replace(".", "__")
-                    if include:
-                        dict_query[f"{mongoengine_field}__in"] = include
-                    if exclude:
-                        dict_query[f"{mongoengine_field}__nin"] = [
-                            t[1:] for t in exclude
-                        ]
+                    query &= cls.get_list_field_query(field, data)
 
             for field in opts.fields or []:
                 data = parameters.pop(field, None)
@@ -229,6 +250,47 @@ class GetMixin(PropsMixin):
 
         return query & RegexQ(**dict_query)
 
+    @classmethod
+    def get_list_field_query(cls, field: str, data: Sequence[Optional[str]]) -> Q:
+        """
+        Get a proper mongoengine Q object that represents an "or" query for the provided values
+        with respect to the given list field, with support for "none of empty" in case a None value
+        is included.
+
+        - Exclusion can be specified by a leading "-" for each value (API versions <2.8)
+            or by a preceding "__$not" value (operator)
+        """
+        if not isinstance(data, (list, tuple)):
+            raise MakeGetAllQueryError("expected list", field)
+
+        # TODO: backwards compatibility only for older API versions
+        helper = cls.ListFieldBucketHelper(legacy=True)
+        actions = bucketize(
+            data, key=helper.key, value_transform=helper.value_transform
+        )
+
+        allow_empty = None in actions.get("in", {})
+        mongoengine_field = field.replace(".", "__")
+
+        q = RegexQ()
+        for action in filter(None, actions):
+            q &= RegexQ(
+                **{
+                    f"{mongoengine_field}__{action}": list(
+                        set(filter(None, actions[action]))
+                    )
+                }
+            )
+
+        if not allow_empty:
+            return q
+
+        return (
+            q
+            | Q(**{f"{mongoengine_field}__exists": False})
+            | Q(**{mongoengine_field: []})
+        )
+
     @classmethod
     def _prepare_perm_query(cls, company, allow_public=False):
         if allow_public:

server/database/model/model.py

@@ -1,7 +1,7 @@
-from mongoengine import Document, StringField, DateTimeField, ListField, BooleanField
+from mongoengine import Document, StringField, DateTimeField, BooleanField
 
 from database import Database, strict
-from database.fields import StrippedStringField, SafeDictField
+from database.fields import StrippedStringField, SafeDictField, SafeSortedListField
 from database.model import DbModelMixin
 from database.model.base import GetMixin
 from database.model.model_labels import ModelLabels
@@ -61,8 +61,8 @@ class Model(DbModelMixin, Document):
     created = DateTimeField(required=True, user_set_allowed=True)
     task = StringField(reference_field=Task)
     comment = StringField(user_set_allowed=True)
-    tags = ListField(StringField(required=True), user_set_allowed=True)
-    system_tags = ListField(StringField(required=True), user_set_allowed=True)
+    tags = SafeSortedListField(StringField(required=True), user_set_allowed=True)
+    system_tags = SafeSortedListField(StringField(required=True), user_set_allowed=True)
     uri = StrippedStringField(default="", user_set_allowed=True)
     framework = StringField()
     design = SafeDictField()

server/database/model/project.py

@@ -1,7 +1,7 @@
-from mongoengine import StringField, DateTimeField, ListField
+from mongoengine import StringField, DateTimeField
 
 from database import Database, strict
-from database.fields import StrippedStringField
+from database.fields import StrippedStringField, SafeSortedListField
 from database.model import AttributedDocument
 from database.model.base import GetMixin
 
@@ -36,7 +36,7 @@ class Project(AttributedDocument):
     )
     description = StringField(required=True)
     created = DateTimeField(required=True)
-    tags = ListField(StringField(required=True))
-    system_tags = ListField(StringField(required=True))
+    tags = SafeSortedListField(StringField(required=True))
+    system_tags = SafeSortedListField(StringField(required=True))
     default_output_destination = StrippedStringField()
     last_update = DateTimeField()

server/database/model/queue.py

@@ -4,11 +4,10 @@ from mongoengine import (
     StringField,
     DateTimeField,
     EmbeddedDocumentListField,
-    ListField,
 )
 
 from database import Database, strict
-from database.fields import StrippedStringField
+from database.fields import StrippedStringField, SafeSortedListField
 from database.model import DbModelMixin
 from database.model.base import ProperDictMixin, GetMixin
 from database.model.company import Company
@@ -41,7 +40,7 @@ class Queue(DbModelMixin, Document):
     )
     company = StringField(required=True, reference_field=Company)
     created = DateTimeField(required=True)
-    tags = ListField(StringField(required=True), default=list, user_set_allowed=True)
-    system_tags = ListField(StringField(required=True), user_set_allowed=True)
+    tags = SafeSortedListField(StringField(required=True), default=list, user_set_allowed=True)
+    system_tags = SafeSortedListField(StringField(required=True), user_set_allowed=True)
     entries = EmbeddedDocumentListField(Entry, default=list)
     last_update = DateTimeField()

server/database/model/task/task.py

@@ -172,8 +172,8 @@ class Task(AttributedDocument):
     project = StringField(reference_field=Project, user_set_allowed=True)
     output = EmbeddedDocumentField(Output, default=Output)
     execution: Execution = EmbeddedDocumentField(Execution, default=Execution)
-    tags = ListField(StringField(required=True), user_set_allowed=True)
-    system_tags = ListField(StringField(required=True), user_set_allowed=True)
+    tags = SafeSortedListField(StringField(required=True), user_set_allowed=True)
+    system_tags = SafeSortedListField(StringField(required=True), user_set_allowed=True)
     script = EmbeddedDocumentField(Script)
     last_worker = StringField()
     last_worker_report = DateTimeField()