From 62d5779bd5b5924f418b8f46d9faca63e119789c Mon Sep 17 00:00:00 2001
From: allegroai <>
Date: Fri, 8 Jul 2022 17:35:01 +0300
Subject: [PATCH] Count own tasks/models for projects

---
 apiserver/bll/project/project_bll.py          | 55 ++++++++++++++----
 apiserver/bll/project/sub_projects.py         |  5 ++
 apiserver/config/default/services/events.conf | 11 ++++
 apiserver/database/model/base.py              | 56 +++++++++++++++----
 apiserver/database/model/project.py           |  4 +-
 apiserver/schema/services/projects.conf       | 32 +++++++----
 apiserver/services/projects.py                |  6 +-
 7 files changed, 133 insertions(+), 36 deletions(-)

diff --git a/apiserver/bll/project/project_bll.py b/apiserver/bll/project/project_bll.py
index 9e82514..38a5671 100644
--- a/apiserver/bll/project/project_bll.py
+++ b/apiserver/bll/project/project_bll.py
@@ -183,6 +183,7 @@ class ProjectBLL:
                 if new_location != old_location:
                     raise errors.bad_request.CannotUpdateProjectLocation(name=new_name)
                 fields["name"] = new_name
+                fields["basename"] = new_name.split("/")[-1]
 
             fields["last_update"] = datetime.utcnow()
             updated = project.update(upsert=False, **fields)
@@ -225,6 +226,7 @@ class ProjectBLL:
             user=user,
             company=company,
             name=name,
+            basename=name.split("/")[-1],
             description=description,
             tags=tags,
             system_tags=system_tags,
@@ -328,6 +330,7 @@ class ProjectBLL:
         project_ids: Sequence[str],
         specific_state: Optional[EntityVisibility] = None,
         filter_: Mapping[str, Any] = None,
+        users: Sequence[str] = None,
     ) -> Tuple[Sequence, Sequence]:
         archived = EntityVisibility.archived.value
 
@@ -352,7 +355,10 @@ class ProjectBLL:
             # count tasks per project per status
             {
                 "$match": cls.get_match_conditions(
-                    company=company_id, project_ids=project_ids, filter_=filter_
+                    company=company_id,
+                    project_ids=project_ids,
+                    filter_=filter_,
+                    users=users,
                 )
             },
             ensure_valid_fields(),
@@ -469,7 +475,10 @@ class ProjectBLL:
             {
                 "$match": {
                     **cls.get_match_conditions(
-                        company=company_id, project_ids=project_ids, filter_=filter_
+                        company=company_id,
+                        project_ids=project_ids,
+                        filter_=filter_,
+                        users=users,
                     ),
                     **get_state_filter(),
                 }
@@ -516,13 +525,18 @@ class ProjectBLL:
         include_children: bool = True,
         search_hidden: bool = False,
         filter_: Mapping[str, Any] = None,
+        users: Sequence[str] = None,
+        user_active_project_ids: Sequence[str] = None,
     ) -> Tuple[Dict[str, dict], Dict[str, dict]]:
         if not project_ids:
             return {}, {}
 
         child_projects = (
             _get_sub_projects(
-                project_ids, _only=("id", "name"), search_hidden=search_hidden
+                project_ids,
+                _only=("id", "name"),
+                search_hidden=search_hidden,
+                allowed_ids=user_active_project_ids,
             )
             if include_children
             else {}
@@ -535,6 +549,7 @@ class ProjectBLL:
             project_ids=list(project_ids_with_children),
             specific_state=specific_state,
             filter_=filter_,
+            users=users,
         )
 
         default_counts = dict.fromkeys(get_options(TaskStatus), 0)
@@ -701,7 +716,7 @@ class ProjectBLL:
         users: Sequence[str],
         project_ids: Optional[Sequence[str]] = None,
         allow_public: bool = True,
-    ) -> Sequence[str]:
+    ) -> Tuple[Sequence[str], Sequence[str]]:
         """
         Get the projects ids where user created any tasks including all the parents of these projects
         If project ids are specified then filter the results by these project ids
@@ -725,13 +740,16 @@ class ProjectBLL:
 
         res = list(res)
         if not res:
-            return res
+            return res, res
 
-        ids_with_parents = _ids_with_parents(res)
-        if project_ids:
-            return [pid for pid in ids_with_parents if pid in project_ids]
+        user_active_project_ids = _ids_with_parents(res)
+        filtered_ids = (
+            list(set(user_active_project_ids) & set(project_ids))
+            if project_ids
+            else list(user_active_project_ids)
+        )
 
-        return ids_with_parents
+        return filtered_ids, user_active_project_ids
 
     @classmethod
     def get_task_parents(
@@ -800,12 +818,18 @@ class ProjectBLL:
 
     @staticmethod
     def get_match_conditions(
-        company: str, project_ids: Sequence[str], filter_: Mapping[str, Any]
+        company: str,
+        project_ids: Sequence[str],
+        filter_: Mapping[str, Any],
+        users: Sequence[str],
     ):
         conditions = {
             "company": {"$in": [None, "", company]},
             "project": {"$in": project_ids},
         }
+        if users:
+            conditions["user"] = {"$in": users}
+
         if not filter_:
             return conditions
 
@@ -828,7 +852,11 @@ class ProjectBLL:
 
     @classmethod
     def calc_own_contents(
-        cls, company: str, project_ids: Sequence[str], filter_: Mapping[str, Any] = None
+        cls,
+        company: str,
+        project_ids: Sequence[str],
+        filter_: Mapping[str, Any] = None,
+        users: Sequence[str] = None,
     ) -> Dict[str, dict]:
         """
         Returns the amount of task/models per requested project
@@ -841,7 +869,10 @@ class ProjectBLL:
         pipeline = [
             {
                 "$match": cls.get_match_conditions(
-                    company=company, project_ids=project_ids, filter_=filter_
+                    company=company,
+                    project_ids=project_ids,
+                    filter_=filter_,
+                    users=users,
                 )
             },
             {"$project": {"project": 1}},
diff --git a/apiserver/bll/project/sub_projects.py b/apiserver/bll/project/sub_projects.py
index 2d2f305..cf7fc7c 100644
--- a/apiserver/bll/project/sub_projects.py
+++ b/apiserver/bll/project/sub_projects.py
@@ -51,6 +51,7 @@ def _ensure_project(
         created=now,
         last_update=now,
         name=name,
+        basename=name.split("/")[-1],
         **(creation_params or dict(description="")),
     )
     parent = _ensure_project(company, user, location, creation_params=creation_params)
@@ -104,6 +105,7 @@ def _get_sub_projects(
     project_ids: Sequence[str],
     _only: Sequence[str] = ("id", "path"),
     search_hidden=True,
+    allowed_ids: Sequence[str] = None,
 ) -> Mapping[str, Sequence[Project]]:
     """
     Return the list of child projects of all the levels for the parent project ids
@@ -111,6 +113,9 @@ def _get_sub_projects(
     query = dict(path__in=project_ids)
     if not search_hidden:
         query["system_tags__nin"] = [EntityVisibility.hidden.value]
+    if allowed_ids:
+        query["id__in"] = allowed_ids
+
     qs = Project.objects(**query)
     if _only:
         _only = set(_only) | {"path"}
diff --git a/apiserver/config/default/services/events.conf b/apiserver/config/default/services/events.conf
index 8e44c3e..a144abf 100644
--- a/apiserver/config/default/services/events.conf
+++ b/apiserver/config/default/services/events.conf
@@ -12,12 +12,23 @@ events_retrieval {
     # should not exceed the amount of concurrent connections set in the ES driver
     max_metrics_concurrency: 4
 
+    # If set then max_metrics_count and max_variants_count are calculated dynamically on user data
+    dynamic_metrics_count: true
+
+    # The percentage from the ES aggs limit (10000) to use for the max_metrics and max_variants calculation
+    dynamic_metrics_count_threshold: 80
+
     # the max amount of metrics to aggregate on
     max_metrics_count: 100
 
     # the max amount of variants to aggregate on
     max_variants_count: 100
 
+    debug_images {
+        # Allow to return the debug images for the variants with uninitialized valid iterations border
+        allow_uninitialized_variants: true
+    }
+
     max_raw_scalars_size: 200000
 
     scroll_id_key: "cTN5VEtWEC6QrHvUl0FTx9kNyO0CcCK1p57akxma"
diff --git a/apiserver/database/model/base.py b/apiserver/database/model/base.py
index 4daf67a..3385749 100644
--- a/apiserver/database/model/base.py
+++ b/apiserver/database/model/base.py
@@ -704,6 +704,45 @@ class GetMixin(PropsMixin):
             v for k, v in cls._field_collation_overrides.items() if field.startswith(k)
         )
 
+    @classmethod
+    def get_count(
+        cls: Union["GetMixin", Document],
+        company,
+        query_dict: dict = None,
+        query_options: QueryParameterOptions = None,
+        query: Q = None,
+        allow_public=False,
+    ) -> int:
+        _query = cls._get_combined_query(
+            company=company,
+            query_dict=query_dict,
+            query_options=query_options,
+            query=query,
+            allow_public=allow_public,
+        )
+        return cls.objects(_query).count()
+
+    @classmethod
+    def _get_combined_query(
+        cls,
+        company,
+        query_dict: dict = None,
+        query_options: QueryParameterOptions = None,
+        query: Q = None,
+        allow_public=False,
+    ) -> Q:
+        if query_dict is not None:
+            q = cls.prepare_query(
+                parameters=query_dict,
+                company=company,
+                parameters_options=query_options,
+                allow_public=allow_public,
+            )
+        else:
+            q = cls._prepare_perm_query(company, allow_public=allow_public)
+
+        return (q & query) if query else q
+
     @classmethod
     def get_many(
         cls,
@@ -749,16 +788,13 @@ class GetMixin(PropsMixin):
                 if override_collation:
                     break
 
-        if query_dict is not None:
-            q = cls.prepare_query(
-                parameters=query_dict,
-                company=company,
-                parameters_options=query_options,
-                allow_public=allow_public,
-            )
-        else:
-            q = cls._prepare_perm_query(company, allow_public=allow_public)
-        _query = (q & query) if query else q
+        _query = cls._get_combined_query(
+            company=company,
+            query_dict=query_dict,
+            query_options=query_options,
+            query=query,
+            allow_public=allow_public,
+        )
 
         if return_dicts:
             data_getter = partial(
diff --git a/apiserver/database/model/project.py b/apiserver/database/model/project.py
index e132947..426dc2e 100644
--- a/apiserver/database/model/project.py
+++ b/apiserver/database/model/project.py
@@ -9,7 +9,7 @@ from apiserver.database.model.base import GetMixin
 class Project(AttributedDocument):
 
     get_all_query_options = GetMixin.QueryParameterOptions(
-        pattern_fields=("name", "description"),
+        pattern_fields=("name", "basename", "description"),
         list_fields=("tags", "system_tags", "id", "parent", "path"),
         range_fields=("last_update",),
     )
@@ -21,6 +21,7 @@ class Project(AttributedDocument):
             "parent",
             "path",
             ("company", "name"),
+            ("company", "basename"),
             {
                 "name": "%s.project.main_text_index" % Database.backend,
                 "fields": ["$name", "$id", "$description"],
@@ -37,6 +38,7 @@ class Project(AttributedDocument):
         min_length=3,
         sparse=True,
     )
+    basename = StrippedStringField(required=True)
     description = StringField()
     created = DateTimeField(required=True)
     tags = SafeSortedListField(StringField(required=True))
diff --git a/apiserver/schema/services/projects.conf b/apiserver/schema/services/projects.conf
index a7f16e5..7a89857 100644
--- a/apiserver/schema/services/projects.conf
+++ b/apiserver/schema/services/projects.conf
@@ -25,6 +25,10 @@ _definitions {
                 description: "Project name"
                 type: string
             }
+            basename {
+                description: "Project base name"
+                type: string
+            }
             description {
                 description: "Project description"
                 type: string
@@ -156,6 +160,10 @@ _definitions {
                 description: "Project name"
                 type: string
             }
+            basename {
+                description: "Project base name"
+                type: string
+            }
             description {
                 description: "Project description"
                 type: string
@@ -214,6 +222,14 @@ _definitions {
                     }
                 }
             }
+            own_tasks {
+                description: "The amount of tasks under this project (without children projects). Returned if 'check_own_contents' flag is set in the request"
+                type: integer
+            }
+            own_models {
+                description: "The amount of models under this project (without children projects). Returned if 'check_own_contents' flag is set in the request"
+                type: integer
+            }
         }
     }
     metric_variant_result {
@@ -385,6 +401,10 @@ get_all {
                     description: "Get only projects whose name matches this pattern (python regular expression syntax)"
                     type: string
                 }
+                basename {
+                    description: "Project base name"
+                    type: string
+                }
                 description {
                     description: "Get only projects whose description matches this pattern (python regular expression syntax)"
                     type: string
@@ -530,18 +550,6 @@ get_all_ex {
                 }
             }
         }
-        response {
-            properties {
-                own_tasks {
-                    description: "The amount of tasks under this project (without children projects). Returned if 'check_own_contents' flag is set in the request"
-                    type: integer
-                }
-                own_models {
-                    description: "The amount of models under this project (without children projects). Returned if 'check_own_contents' flag is set in the request"
-                    type: integer
-                }
-            }
-        }
     }
     "2.14": ${get_all_ex."2.13"} {
         request.properties.search_hidden {
diff --git a/apiserver/services/projects.py b/apiserver/services/projects.py
index 3991f89..eb692b0 100644
--- a/apiserver/services/projects.py
+++ b/apiserver/services/projects.py
@@ -110,8 +110,9 @@ def get_all_ex(call: APICall, company_id: str, request: ProjectsGetRequest):
         data, shallow_search=request.shallow_search,
     )
     with TimingContext("mongo", "projects_get_all"):
+        user_active_project_ids = None
         if request.active_users:
-            ids = project_bll.get_projects_with_active_user(
+            ids, user_active_project_ids = project_bll.get_projects_with_active_user(
                 company=company_id,
                 users=request.active_users,
                 project_ids=requested_ids,
@@ -139,6 +140,7 @@ def get_all_ex(call: APICall, company_id: str, request: ProjectsGetRequest):
                     company=company_id,
                     project_ids=list(existing_requested_ids),
                     filter_=request.include_stats_filter,
+                    users=request.active_users,
                 )
                 for project in projects:
                     project.update(**contents.get(project["id"], {}))
@@ -156,6 +158,8 @@ def get_all_ex(call: APICall, company_id: str, request: ProjectsGetRequest):
             include_children=request.stats_with_children,
             search_hidden=request.search_hidden,
             filter_=request.include_stats_filter,
+            users=request.active_users,
+            user_active_project_ids=user_active_project_ids,
         )
 
         for project in projects: