Version bump to v1.17.0

For the num of requests serving charts always take the max value from the interval

LICENSE

@@ -1,7 +1,7 @@
                      Server Side Public License
                      VERSION 1, OCTOBER 16, 2018
 
-                    Copyright © 2019 allegro.ai, Inc.
+                    Copyright © 2024 ClearML Inc.
 
   Everyone is permitted to copy and distribute verbatim copies of this
   license document, but changing it is not allowed.

README.md

@@ -6,7 +6,7 @@
 </br>Experiment Manager, ML-Ops and Data-Management**
 
 [![GitHub license](https://img.shields.io/badge/license-SSPL-green.svg)](https://img.shields.io/badge/license-SSPL-green.svg)
-[![Python versions](https://img.shields.io/badge/python-3.6%20%7C%203.7-blue.svg)](https://img.shields.io/badge/python-3.6%20%7C%203.7-blue.svg)
+[![Python versions](https://img.shields.io/badge/python-3.9-blue.svg)](https://img.shields.io/badge/python-3.9-blue.svg)
 [![GitHub version](https://img.shields.io/github/release-pre/allegroai/trains-server.svg)](https://img.shields.io/github/release-pre/allegroai/trains-server.svg)
 [![Artifact Hub](https://img.shields.io/endpoint?url=https://artifacthub.io/badge/repository/allegroai)](https://artifacthub.io/packages/search?repo=allegroai)
 

apiserver/LICENSE

@@ -1,7 +1,7 @@
                      Server Side Public License
                      VERSION 1, OCTOBER 16, 2018
 
-                    Copyright © 2019 allegro.ai, Inc.
+                    Copyright © 2024 ClearML Inc.
 
   Everyone is permitted to copy and distribute verbatim copies of this
   license document, but changing it is not allowed.

apiserver/apierrors/errors.conf

@@ -84,6 +84,7 @@
     411: ["project_cannot_be_moved_under_itself", "Project can not be moved under itself in the projects hierarchy"]
     412: ["project_cannot_be_merged_into_its_child", "Project can not be merged into its own child"]
     413: ["project_has_pipelines", "project has associated pipelines with active controllers"]
+    414: ["public_project_exists", "Cannot create project. Public project with the same name already exists"]
 
     # Queues
     701: ["invalid_queue_id", "invalid queue id"]
@@ -106,6 +107,11 @@
     1004: ["worker_not_registered", "worker is not registered"]
     1005: ["worker_stats_not_found", "worker stats not found"]
 
+    # Serving
+    1050: ["invalid_container_id", "invalid container id"]
+    1051: ["container_not_registered", "container is not registered"]
+    1052: ["no_containers_for_url", "no container instances found for serice url"]
+
     1104: ["invalid_scroll_id", "Invalid scroll id"]
 }
 

apiserver/apimodels/__init__.py

@@ -1,10 +1,11 @@
 from enum import Enum
-from typing import Union, Type, Iterable
+from numbers import Number
+from typing import Union, Type, Iterable, Mapping
 
 import jsonmodels.errors
 import six
 from jsonmodels import fields
-from jsonmodels.fields import _LazyType, NotSet
+from jsonmodels.fields import _LazyType, NotSet, EmbeddedField
 from jsonmodels.models import Base as ModelBase
 from jsonmodels.validators import Enum as EnumValidator
 from mongoengine.base import BaseDocument
@@ -40,6 +41,34 @@ def make_default(field_cls, default_value):
     return _FieldWithDefault
 
 
+class OneOfEmbeddedField(EmbeddedField):
+    def __init__(
+        self,
+        *args,
+        discriminator_property: str,
+        discriminator_mapping: Mapping[str, type],
+        **kwargs,
+    ):
+        self.discriminator_property = discriminator_property
+        self.discriminator_mapping = discriminator_mapping
+        model_types = tuple(set(self.discriminator_mapping.values()))
+
+        super().__init__(model_types, *args, **kwargs)
+
+    def parse_value(self, value):
+        """Parse value to proper model type."""
+        if not isinstance(value, dict) or self.discriminator_property not in value:
+            return super().parse_value(value)
+
+        property_value = value.get(self.discriminator_property)
+        embed_type = self.discriminator_mapping.get(property_value)
+        if not embed_type:
+            raise jsonmodels.errors.ValidationError(
+                f"Could not find type matching discriminator property value: {property_value}"
+            )
+        return embed_type(**value)
+
+
 class ListField(fields.ListField):
     def __init__(self, items_types=None, *args, default=NotSet, **kwargs):
         if default is not NotSet and callable(default):
@@ -68,6 +97,15 @@ class ScalarField(fields.BaseField):
     types = (str, int, float, bool)
 
 
+class SafeStringField(fields.StringField):
+    """String field that can also accept numbers as input"""
+    def parse_value(self, value):
+        if isinstance(value, Number):
+            value = str(value)
+
+        return super().parse_value(value)
+
+
 class DictField(fields.BaseField):
     types = (dict,)
 
@@ -115,9 +153,7 @@ class DictField(fields.BaseField):
             if len(self.value_types) != 1:
                 tpl = 'Cannot decide which type to choose from "{types}".'
                 raise jsonmodels.errors.ValidationError(
-                    tpl.format(
-                        types=', '.join([t.__name__ for t in self.value_types])
-                    )
+                    tpl.format(types=", ".join([t.__name__ for t in self.value_types]))
                 )
             return self.value_types[0](**value)
 
@@ -179,7 +215,7 @@ class EnumField(fields.StringField):
         *args,
         required=False,
         default=None,
-        **kwargs
+        **kwargs,
     ):
         choices = list(map(self.parse_value, values_or_type))
         validator_cls = EnumValidator if required else NullableEnumValidator
@@ -202,7 +238,7 @@ class ActualEnumField(fields.StringField):
         validators=None,
         required=False,
         default=None,
-        **kwargs
+        **kwargs,
     ):
         self.__enum = enum_class
         self.types = (enum_class,)
@@ -215,7 +251,7 @@ class ActualEnumField(fields.StringField):
             *args,
             required=required,
             validators=validators,
-            **kwargs
+            **kwargs,
         )
 
     def parse_value(self, value):

apiserver/apimodels/events.py

@@ -13,6 +13,14 @@ from apiserver.config_repo import config
 from apiserver.utilities.stringenum import StringEnum
 
 
+class TaskRequest(Base):
+    task: str = StringField(required=True)
+
+
+class ModelRequest(Base):
+    model: str = StringField(required=True)
+
+
 class HistogramRequestBase(Base):
     samples: int = IntField(default=2000, validators=[Min(1), Max(6000)])
     key: ScalarKeyEnum = ActualEnumField(ScalarKeyEnum, default=ScalarKeyEnum.iter)
@@ -29,6 +37,11 @@ class ScalarMetricsIterHistogramRequest(HistogramRequestBase):
     model_events: bool = BoolField(default=False)
 
 
+class GetMetricsAndVariantsRequest(Base):
+    task: str = StringField(required=True)
+    model_events: bool = BoolField(default=False)
+
+
 class MultiTaskScalarMetricsIterHistogramRequest(HistogramRequestBase):
     tasks: Sequence[str] = ListField(
         items_types=str,
@@ -41,6 +54,7 @@ class MultiTaskScalarMetricsIterHistogramRequest(HistogramRequestBase):
             )
         ],
     )
+    metrics: Sequence[MetricVariants] = ListField(items_types=MetricVariants)
     model_events: bool = BoolField(default=False)
 
 
@@ -50,6 +64,12 @@ class TaskMetric(Base):
     variants: Sequence[str] = ListField(items_types=str)
 
 
+class LegacyMetricEventsRequest(TaskRequest):
+    iters: int = IntField(default=1, validators=validators.Min(1))
+    scroll_id: str = StringField()
+    model_events: bool = BoolField(default=False)
+
+
 class MetricEventsRequest(Base):
     metrics: Sequence[TaskMetric] = ListField(
         items_types=TaskMetric, validators=[Length(minimum_value=1)]
@@ -58,7 +78,14 @@ class MetricEventsRequest(Base):
     navigate_earlier: bool = BoolField(default=True)
     refresh: bool = BoolField(default=False)
     scroll_id: str = StringField()
-    model_events: bool = BoolField()
+    model_events: bool = BoolField(default=False)
+
+
+class VectorMetricsIterHistogramRequest(Base):
+    task: str = StringField(required=True)
+    metric: str = StringField(required=True)
+    variant: str = StringField(required=True)
+    model_events: bool = BoolField(default=False)
 
 
 class GetVariantSampleRequest(Base):
@@ -109,11 +136,17 @@ class TaskEventsRequest(TaskEventsRequestBase):
     model_events: bool = BoolField(default=False)
 
 
+class LegacyLogEventsRequest(TaskEventsRequestBase):
+    order: Optional[str] = ActualEnumField(LogOrderEnum, default=LogOrderEnum.desc)
+    scroll_id: str = StringField()
+
+
 class LogEventsRequest(TaskEventsRequestBase):
     batch_size: int = IntField(default=5000)
     navigate_earlier: bool = BoolField(default=True)
     from_timestamp: Optional[int] = IntField()
     order: Optional[str] = ActualEnumField(LogOrderEnum)
+    metrics: Sequence[MetricVariants] = ListField(items_types=MetricVariants)
 
 
 class ScalarMetricsIterRawRequest(TaskEventsRequestBase):
@@ -148,18 +181,28 @@ class MultiTasksRequestBase(Base):
 
 
 class SingleValueMetricsRequest(MultiTasksRequestBase):
-    pass
+    metrics: Sequence[MetricVariants] = ListField(items_types=MetricVariants)
 
 
 class TaskMetricsRequest(MultiTasksRequestBase):
     event_type: EventType = ActualEnumField(EventType, required=True)
 
 
+class MultiTaskMetricsRequest(MultiTasksRequestBase):
+    event_type: EventType = ActualEnumField(EventType, default=EventType.all)
+
+
+class LegacyMultiTaskEventsRequest(MultiTasksRequestBase):
+    iters: int = IntField(default=1, validators=validators.Min(1))
+    scroll_id: str = StringField()
+
+
 class MultiTaskPlotsRequest(MultiTasksRequestBase):
     iters: int = IntField(default=1)
     scroll_id: str = StringField()
     no_scroll: bool = BoolField(default=False)
     last_iters_per_task_metric: bool = BoolField(default=True)
+    metrics: Sequence[MetricVariants] = ListField(items_types=MetricVariants)
 
 
 class TaskPlotsRequest(Base):
@@ -171,6 +214,14 @@ class TaskPlotsRequest(Base):
     model_events: bool = BoolField(default=False)
 
 
+class GetScalarMetricDataRequest(Base):
+    task: str = StringField(required=True)
+    metric: str = StringField(required=True)
+    scroll_id: str = StringField()
+    no_scroll: bool = BoolField(default=False)
+    model_events: bool = BoolField(default=False)
+
+
 class ClearScrollRequest(Base):
     scroll_id: str = StringField()
 
@@ -179,3 +230,5 @@ class ClearTaskLogRequest(Base):
     task: str = StringField(required=True)
     threshold_sec = IntField()
     allow_locked = BoolField(default=False)
+    exclude_metrics = ListField(items_types=[str])
+    include_metrics = ListField(items_types=[str])

apiserver/apimodels/login.py

@@ -5,8 +5,9 @@ from apiserver.apimodels import DictField, callable_default
 
 
 class GetSupportedModesRequest(Base):
-    state = StringField(help_text="ASCII base64 encoded application state")
-    callback_url_prefix = StringField()
+    pass
+    # state = StringField(help_text="ASCII base64 encoded application state")
+    # callback_url_prefix = StringField()
 
 
 class BasicGuestMode(Base):

apiserver/apimodels/models.py

@@ -42,6 +42,21 @@ class ModelRequest(models.Base):
     model = fields.StringField(required=True)
 
 
+class TaskRequest(models.Base):
+    task = fields.StringField(required=True)
+
+
+class UpdateForTaskRequest(TaskRequest):
+    uri = fields.StringField()
+    iteration = fields.IntField()
+    override_model_id = fields.StringField()
+
+
+class UpdateModelRequest(ModelRequest):
+    task = fields.StringField()
+    iteration = fields.IntField()
+
+
 class DeleteModelRequest(ModelRequest):
     force = fields.BoolField(default=False)
     delete_external_artifacts = fields.BoolField(default=True)

apiserver/apimodels/pipelines.py

@@ -18,8 +18,4 @@ class StartPipelineRequest(models.Base):
     task = fields.StringField(required=True)
     queue = fields.StringField(required=True)
     args = ListField(Arg)
-
-
-class StartPipelineResponse(models.Base):
-    pipeline = fields.StringField(required=True)
-    enqueued = fields.BoolField(required=True)
+    verify_watched_queue = fields.BoolField(default=False)

apiserver/apimodels/projects.py

@@ -33,6 +33,7 @@ class ProjectOrNoneRequest(models.Base):
 
 class GetUniqueMetricsRequest(ProjectOrNoneRequest):
     model_metrics = fields.BoolField(default=False)
+    ids = fields.ListField(str)
 
 
 class GetParamsRequest(ProjectOrNoneRequest):
@@ -45,7 +46,7 @@ class ProjectTagsRequest(TagsRequest):
 
 
 class MultiProjectRequest(models.Base):
-    projects = fields.ListField(str)
+    projects = fields.ListField(items_types=[str, type(None)])
     include_subprojects = fields.BoolField(default=True)
 
 
@@ -72,6 +73,7 @@ class MultiProjectPagedRequest(MultiProjectRequest):
 class ProjectHyperparamValuesRequest(MultiProjectPagedRequest):
     section = fields.StringField(required=True)
     name = fields.StringField(required=True)
+    pattern = fields.StringField()
 
 
 class ProjectModelMetadataValuesRequest(MultiProjectPagedRequest):
@@ -98,3 +100,4 @@ class ProjectsGetRequest(models.Base):
     allow_public = fields.BoolField(default=True)
     children_type = ActualEnumField(ProjectChildrenType)
     children_tags = fields.ListField(str)
+    children_tags_filter = DictField()

apiserver/apimodels/queues.py

@@ -56,6 +56,14 @@ class TaskRequest(QueueRequest):
     task = StringField(required=True)
 
 
+class RemoveTaskRequest(TaskRequest):
+    update_task_status = BoolField(default=False)
+
+
+class AddTaskRequest(TaskRequest):
+    update_execution_queue = BoolField(default=True)
+
+
 class MoveTaskRequest(TaskRequest):
     count = IntField(default=1)
 

apiserver/apimodels/server.py

@@ -6,6 +6,10 @@ class ReportStatsOptionRequest(Base):
     enabled = BoolField(default=None, nullable=True)
 
 
+class GetConfigRequest(Base):
+    path = StringField()
+
+
 class ReportStatsOptionResponse(Base):
     supported = BoolField(default=True)
     enabled = BoolField()

apiserver/apimodels/serving.py

@@ -0,0 +1,104 @@
+from enum import Enum
+from typing import Sequence
+
+from jsonmodels.models import Base
+from jsonmodels.fields import (
+    StringField,
+    EmbeddedField,
+    DateTimeField,
+    IntField,
+    FloatField,
+    BoolField,
+)
+from jsonmodels import validators
+from jsonmodels.validators import Min
+
+from apiserver.apimodels import ListField, JsonSerializableMixin, SafeStringField
+from apiserver.apimodels import ActualEnumField
+from apiserver.config_repo import config
+from .workers import MachineStats
+
+
+class ReferenceItem(Base):
+    type = StringField(
+        required=True,
+        validators=validators.Enum("app_id", "app_instance", "model", "task", "url"),
+    )
+    value = StringField(required=True)
+
+
+class ServingModel(Base):
+    container_id = StringField(required=True)
+    endpoint_name = StringField(required=True)
+    endpoint_url = StringField()  # can be not existing yet at registration time
+    model_name = StringField(required=True)
+    model_source = StringField()
+    model_version = StringField()
+    preprocess_artifact = StringField()
+    input_type = StringField()
+    input_size = SafeStringField()
+    tags = ListField(str)
+    system_tags = ListField(str)
+    reference: Sequence[ReferenceItem] = ListField(ReferenceItem)
+
+
+class RegisterRequest(ServingModel):
+    timeout = IntField(
+        default=int(
+            config.get("services.serving.default_container_timeout_sec", 10 * 60)
+        ),
+        validators=[Min(1)],
+    )
+    """ registration timeout in seconds (default is 10min) """
+
+
+class UnregisterRequest(Base):
+    container_id = StringField(required=True)
+
+
+class StatusReportRequest(ServingModel):
+    uptime_sec = IntField()
+    requests_num = IntField()
+    requests_min = FloatField()
+    latency_ms = IntField()
+    machine_stats: MachineStats = EmbeddedField(MachineStats)
+
+
+class ServingContainerEntry(StatusReportRequest, JsonSerializableMixin):
+    key = StringField(required=True)
+    company_id = StringField(required=True)
+    ip = StringField()
+    register_time = DateTimeField(required=True)
+    register_timeout = IntField(required=True)
+    last_activity_time = DateTimeField(required=True)
+
+
+class GetEndpointDetailsRequest(Base):
+    endpoint_url = StringField(required=True)
+
+
+class MetricType(Enum):
+    requests = "requests"
+    requests_min = "requests_min"
+    latency_ms = "latency_ms"
+    cpu_count = "cpu_count"
+    gpu_count = "gpu_count"
+    cpu_util = "cpu_util"
+    gpu_util = "gpu_util"
+    ram_total = "ram_total"
+    ram_used = "ram_used"
+    ram_free = "ram_free"
+    gpu_ram_total = "gpu_ram_total"
+    gpu_ram_used = "gpu_ram_used"
+    gpu_ram_free = "gpu_ram_free"
+    network_rx = "network_rx"
+    network_tx = "network_tx"
+
+
+class GetEndpointMetricsHistoryRequest(Base):
+    from_date = FloatField(required=True, validators=Min(0))
+    to_date = FloatField(required=True, validators=Min(0))
+    interval = IntField(required=True, validators=Min(1))
+    endpoint_url = StringField(required=True)
+    metric_type = ActualEnumField(MetricType, default=MetricType.requests)
+    instance_charts = BoolField(default=True)

apiserver/apimodels/storage.py

@@ -0,0 +1,60 @@
+from jsonmodels.fields import StringField, BoolField, ListField, EmbeddedField
+from jsonmodels.models import Base
+from jsonmodels.validators import Enum
+
+
+class AWSBucketSettings(Base):
+    bucket = StringField()
+    subdir = StringField()
+    host = StringField()
+    key = StringField()
+    secret = StringField()
+    token = StringField()
+    multipart = BoolField(default=True)
+    acl = StringField()
+    secure = BoolField(default=True)
+    region = StringField()
+    verify = BoolField(default=True)
+    use_credentials_chain = BoolField(default=False)
+
+
+class AWSSettings(Base):
+    key = StringField()
+    secret = StringField()
+    region = StringField()
+    token = StringField()
+    use_credentials_chain = BoolField(default=False)
+    buckets = ListField(items_types=[AWSBucketSettings])
+
+
+class GoogleBucketSettings(Base):
+    bucket = StringField()
+    subdir = StringField()
+    project = StringField()
+    credentials_json = StringField()
+
+
+class GoogleSettings(Base):
+    project = StringField()
+    credentials_json = StringField()
+    buckets = ListField(items_types=[GoogleBucketSettings])
+
+
+class AzureContainerSettings(Base):
+    account_name = StringField()
+    account_key = StringField()
+    container_name = StringField()
+
+
+class AzureSettings(Base):
+    containers = ListField(items_types=[AzureContainerSettings])
+
+
+class SetSettingsRequest(Base):
+    aws = EmbeddedField(AWSSettings)
+    google = EmbeddedField(GoogleSettings)
+    azure = EmbeddedField(AzureSettings)
+
+
+class ResetSettingsRequest(Base):
+    keys = ListField([str], item_validators=[Enum("aws", "google", "azure")])

apiserver/apimodels/tasks.py

@@ -101,10 +101,15 @@ class DequeueRequest(UpdateRequest):
     new_status = StringField()
 
 
+class StopRequest(UpdateRequest):
+    include_pipeline_steps = BoolField(default=False)
+
+
 class EnqueueRequest(UpdateRequest):
     queue = StringField()
     queue_name = StringField()
     verify_watched_queue = BoolField(default=False)
+    update_execution_queue = BoolField(default=True)
 
 
 class DeleteRequest(UpdateRequest):
@@ -112,6 +117,7 @@ class DeleteRequest(UpdateRequest):
     return_file_urls = BoolField(default=False)
     delete_output_models = BoolField(default=True)
     delete_external_artifacts = BoolField(default=True)
+    include_pipeline_steps = BoolField(default=False)
 
 
 class SetRequirementsRequest(TaskRequest):
@@ -264,6 +270,7 @@ class DeleteConfigurationRequest(TaskUpdateRequest):
 class ArchiveRequest(MultiTaskRequest):
     status_reason = StringField(default="")
     status_message = StringField(default="")
+    include_pipeline_steps = BoolField(default=False)
 
 
 class ArchiveResponse(models.Base):
@@ -275,8 +282,17 @@ class TaskBatchRequest(BatchRequest):
     status_message = StringField(default="")
 
 
+class ArchiveManyRequest(TaskBatchRequest):
+    include_pipeline_steps = BoolField(default=False)
+
+
+class UnarchiveManyRequest(TaskBatchRequest):
+    include_pipeline_steps = BoolField(default=False)
+
+
 class StopManyRequest(TaskBatchRequest):
     force = BoolField(default=False)
+    include_pipeline_steps = BoolField(default=False)
 
 
 class DequeueManyRequest(TaskBatchRequest):
@@ -297,6 +313,7 @@ class DeleteManyRequest(TaskBatchRequest):
     delete_output_models = BoolField(default=True)
     force = BoolField(default=False)
     delete_external_artifacts = BoolField(default=True)
+    include_pipeline_steps = BoolField(default=False)
 
 
 class ResetManyRequest(TaskBatchRequest):
@@ -333,3 +350,8 @@ class DeleteModelsRequest(TaskRequest):
 class GetAllReq(models.Base):
     allow_public = BoolField(default=True)
     search_hidden = BoolField(default=False)
+
+
+class UpdateTagsRequest(BatchRequest):
+    add_tags = ListField([str])
+    remove_tags = ListField([str])

apiserver/apimodels/users.py

@@ -4,6 +4,10 @@ from jsonmodels.models import Base
 from apiserver.apimodels import DictField
 
 
+class UserRequest(Base):
+    user = StringField(required=True)
+
+
 class CreateRequest(Base):
     id = StringField(required=True)
     name = StringField(required=True)

apiserver/apimodels/workers.py

@@ -13,8 +13,7 @@ from jsonmodels.fields import (
 from jsonmodels.models import Base
 
 from apiserver.apimodels import ListField, EnumField, JsonSerializableMixin
-
-DEFAULT_TIMEOUT = 10 * 60
+from apiserver.config_repo import config
 
 
 class WorkerRequest(Base):
@@ -24,7 +23,10 @@ class WorkerRequest(Base):
 
 
 class RegisterRequest(WorkerRequest):
-    timeout = IntField(default=0)  # registration timeout in seconds (if not specified, default is 10min)
+    timeout = IntField(
+        default=int(config.get("services.workers.default_worker_timeout_sec", 10 * 60))
+    )
+    """ registration timeout in seconds (default is 10min) """
     queues = ListField(six.string_types)  # list of queues this worker listens to
 
 
@@ -98,6 +100,7 @@ class GetAllRequest(Base):
     last_seen = IntField(default=3600)
     tags = ListField(str)
     system_tags = ListField(str)
+    worker_pattern = StringField()
 
 
 class GetAllResponse(Base):

apiserver/bll/event/event_bll.py

@@ -5,7 +5,6 @@ import zlib
 from collections import defaultdict
 from contextlib import closing
 from datetime import datetime
-from operator import attrgetter
 from typing import Sequence, Set, Tuple, Optional, List, Mapping, Union
 
 import elasticsearch
@@ -24,6 +23,7 @@ from apiserver.bll.event.event_common import (
     get_metric_variants_condition,
     uncompress_plot,
     get_max_metric_and_variant_counts,
+    PlotFields,
 )
 from apiserver.bll.event.events_iterator import EventsIterator, TaskEventsResult
 from apiserver.bll.event.history_debug_image_iterator import HistoryDebugImageIterator
@@ -31,6 +31,7 @@ from apiserver.bll.event.history_plots_iterator import HistoryPlotsIterator
 from apiserver.bll.event.metric_debug_images_iterator import MetricDebugImagesIterator
 from apiserver.bll.event.metric_plots_iterator import MetricPlotsIterator
 from apiserver.bll.model import ModelBLL
+from apiserver.bll.task.utils import get_many_tasks_for_writing
 from apiserver.bll.util import parallel_chunked_decorator
 from apiserver.database import utils as dbutils
 from apiserver.database.model.model import Model
@@ -40,28 +41,24 @@ from apiserver.bll.event.event_metrics import EventMetrics
 from apiserver.bll.task import TaskBLL
 from apiserver.config_repo import config
 from apiserver.database.errors import translate_errors_context
-from apiserver.database.model.task.task import Task, TaskStatus
+from apiserver.database.model.task.task import TaskStatus
 from apiserver.redis_manager import redman
-from apiserver.tools import safe_get
+from apiserver.service_repo.auth import Identity
 from apiserver.utilities.dicts import nested_get
 from apiserver.utilities.json import loads
 
 # noinspection PyTypeChecker
-EVENT_TYPES: Set[str] = set(map(attrgetter("value"), EventType))
+EVENT_TYPES: Set[str] = set(et.value for et in EventType if et != EventType.all)
 LOCKED_TASK_STATUSES = (TaskStatus.publishing, TaskStatus.published)
-MAX_LONG = 2 ** 63 - 1
-MIN_LONG = -(2 ** 63)
+MAX_LONG = 2**63 - 1
+MIN_LONG = -(2**63)
 
 
 log = config.logger(__file__)
-
-
-class PlotFields:
-    valid_plot = "valid_plot"
-    plot_len = "plot_len"
-    plot_str = "plot_str"
-    plot_data = "plot_data"
-    source_urls = "source_urls"
+async_task_events_delete = config.get("services.tasks.async_events_delete", False)
+async_delete_threshold = config.get(
+    "services.tasks.async_events_delete_threshold", 100_000
+)
 
 
 class EventBLL(object):
@@ -103,7 +100,9 @@ class EventBLL(object):
         return self._metrics
 
     @staticmethod
-    def _get_valid_entities(company_id, ids: Mapping[str, bool], model=False) -> Set:
+    def _get_valid_entities(
+        company_id, ids: Mapping[str, bool], identity: Identity, model=False
+    ) -> Set:
         """Verify that task or model exists and can be updated"""
         if not ids:
             return set()
@@ -122,16 +121,34 @@ class EventBLL(object):
             ):
                 if not requested_ids:
                     continue
-                query = Q(id__in=requested_ids, company=company_id)
-                res.update(
-                    (Model if model else Task).objects(query & locked_q).scalar("id")
-                )
+
+                query = Q(id__in=requested_ids) & locked_q
+                if model:
+                    ids = Model.objects(query & Q(company=company_id)).scalar("id")
+                else:
+                    ids = {
+                        t.id
+                        for t in get_many_tasks_for_writing(
+                            company_id=company_id,
+                            identity=identity,
+                            query=query,
+                            only=("id",),
+                            throw_on_forbidden=False,
+                        )
+                    }
+
+                res.update(ids)
 
             return res
 
     def add_events(
-        self, company_id, events, worker
+        self,
+        company_id: str,
+        identity: Identity,
+        events: Sequence[dict],
+        worker: str,
     ) -> Tuple[int, int, dict]:
+        user_id = identity.user
         task_ids = {}
         model_ids = {}
         for event in events:
@@ -163,8 +180,12 @@ class EventBLL(object):
                 "Inconsistent model_event setting in the passed events",
                 tasks=found_in_both,
             )
-        valid_models = self._get_valid_entities(company_id, ids=model_ids, model=True)
-        valid_tasks = self._get_valid_entities(company_id, ids=task_ids)
+        valid_models = self._get_valid_entities(
+            company_id, ids=model_ids, identity=identity, model=True
+        )
+        valid_tasks = self._get_valid_entities(
+            company_id, ids=task_ids, identity=identity
+        )
 
         actions: List[dict] = []
         used_task_ids = set()
@@ -268,11 +289,13 @@ class EventBLL(object):
             else:
                 used_task_ids.add(task_or_model_id)
                 self._update_last_metric_events_for_task(
-                    last_events=task_last_events[task_or_model_id], event=event,
+                    last_events=task_last_events[task_or_model_id],
+                    event=event,
                 )
             if event_type == EventType.metrics_scalar.value:
                 self._update_last_scalar_events_for_task(
-                    last_events=task_last_scalar_events[task_or_model_id], event=event,
+                    last_events=task_last_scalar_events[task_or_model_id],
+                    event=event,
                 )
 
             actions.append(es_action)
@@ -296,6 +319,7 @@ class EventBLL(object):
             if actions:
                 chunk_size = 500
                 # TODO: replace it with helpers.parallel_bulk in the future once the parallel pool leak is fixed
+                # noinspection PyTypeChecker
                 with closing(
                     elasticsearch.helpers.streaming_bulk(
                         self.es,
@@ -311,20 +335,23 @@ class EventBLL(object):
                         else:
                             errors_per_type["Error when indexing events batch"] += 1
 
+            now = datetime.utcnow()
             for model_id in used_model_ids:
                 ModelBLL.update_statistics(
                     company_id=company_id,
+                    user_id=user_id,
                     model_id=model_id,
+                    last_update=now,
                     last_iteration_max=task_iteration.get(model_id),
                     last_scalar_events=task_last_scalar_events.get(model_id),
                 )
             remaining_tasks = set()
-            now = datetime.utcnow()
             for task_id in used_task_ids:
                 # Update related tasks. For reasons of performance, we prefer to update
                 # all of them and not only those who's events were successful
                 updated = self._update_task(
                     company_id=company_id,
+                    user_id=user_id,
                     task_id=task_id,
                     now=now,
                     iter_max=task_iteration.get(task_id),
@@ -336,14 +363,19 @@ class EventBLL(object):
                     continue
 
             if remaining_tasks:
-                TaskBLL.set_last_update(remaining_tasks, company_id, last_update=now)
+                TaskBLL.set_last_update(
+                    remaining_tasks,
+                    company_id=company_id,
+                    user_id=user_id,
+                    last_update=now,
+                )
 
             # this is for backwards compatibility with streaming bulk throwing exception on those
             invalid_iterations_count = errors_per_type.get(invalid_iteration_error)
             if invalid_iterations_count:
                 raise BulkIndexError(
                     f"{invalid_iterations_count} document(s) failed to index.",
-                    [invalid_iteration_error],
+                    [{"_index": invalid_iteration_error}],
                 )
 
         if not added:
@@ -407,47 +439,45 @@ class EventBLL(object):
         last_events contains [hashed_metric_name -> hashed_variant_name -> event]. Keys are hashed to avoid mongodb
         key conflicts due to invalid characters and/or long field names.
         """
-        metric = event.get("metric")
-        variant = event.get("variant")
-        if not (metric and variant):
+        value = event.get("value")
+        if value is None:
             return
 
+        metric = event.get("metric") or ""
+        variant = event.get("variant") or ""
         metric_hash = dbutils.hash_field_name(metric)
         variant_hash = dbutils.hash_field_name(variant)
 
         last_event = last_events[metric_hash][variant_hash]
+        last_event["metric"] = metric
+        last_event["variant"] = variant
+        last_event["count"] = last_event.get("count", 0) + 1
+        last_event["total"] = last_event.get("total", 0) + value
+
         event_iter = event.get("iter", 0)
         event_timestamp = event.get("timestamp", 0)
-        value = event.get("value")
-        if value is not None and (
-            (event_iter, event_timestamp)
-            >= (
-                last_event.get("iter", event_iter),
-                last_event.get("timestamp", event_timestamp),
-            )
+        if (event_iter, event_timestamp) >= (
+            last_event.get("iter", event_iter),
+            last_event.get("timestamp", event_timestamp),
         ):
-            event_data = {
-                k: event[k]
-                for k in ("value", "metric", "variant", "iter", "timestamp")
-                if k in event
-            }
-            last_event_min_value = last_event.get("min_value", value)
-            last_event_min_value_iter = last_event.get("min_value_iter", event_iter)
-            if value < last_event_min_value:
-                event_data["min_value"] = value
-                event_data["min_value_iter"] = event_iter
-            else:
-                event_data["min_value"] = last_event_min_value
-                event_data["min_value_iter"] = last_event_min_value_iter
-            last_event_max_value = last_event.get("max_value", value)
-            last_event_max_value_iter = last_event.get("max_value_iter", event_iter)
-            if value > last_event_max_value:
-                event_data["max_value"] = value
-                event_data["max_value_iter"] = event_iter
-            else:
-                event_data["max_value"] = last_event_max_value
-                event_data["max_value_iter"] = last_event_max_value_iter
-            last_events[metric_hash][variant_hash] = event_data
+            last_event["value"] = value
+            last_event["iter"] = event_iter
+            last_event["timestamp"] = event_timestamp
+
+        first_value_iter = last_event.get("first_value_iter")
+        if first_value_iter is None or event_iter < first_value_iter:
+            last_event["first_value"] = value
+            last_event["first_value_iter"] = event_iter
+
+        last_event_min_value = last_event.get("min_value")
+        if last_event_min_value is None or value < last_event_min_value:
+            last_event["min_value"] = value
+            last_event["min_value_iter"] = event_iter
+
+        last_event_max_value = last_event.get("max_value")
+        if last_event_max_value is None or value > last_event_max_value:
+            last_event["max_value"] = value
+            last_event["max_value_iter"] = event_iter
 
     def _update_last_metric_events_for_task(self, last_events, event):
         """
@@ -455,9 +485,9 @@ class EventBLL(object):
         recent than the currently stored event for its metric/event_type combination.
         last_events contains [metric_name -> event_type -> event]
         """
-        metric = event.get("metric")
+        metric = event.get("metric") or ""
         event_type = event.get("type")
-        if not (metric and event_type):
+        if not event_type:
             return
 
         timestamp = last_events[metric][event_type].get("timestamp", None)
@@ -466,9 +496,10 @@ class EventBLL(object):
 
     def _update_task(
         self,
-        company_id,
-        task_id,
-        now,
+        company_id: str,
+        user_id: str,
+        task_id: str,
+        now: datetime,
         iter_max=None,
         last_scalar_events=None,
         last_events=None,
@@ -484,8 +515,9 @@ class EventBLL(object):
             return False
 
         return TaskBLL.update_statistics(
-            task_id,
-            company_id,
+            task_id=task_id,
+            company_id=company_id,
+            user_id=user_id,
             last_update=now,
             last_iteration_max=iter_max,
             last_scalar_events=last_scalar_events,
@@ -569,7 +601,8 @@ class EventBLL(object):
         query = {"bool": {"must": must}}
         search_args = dict(es=self.es, company_id=company_id, event_type=event_type)
         max_metrics, max_variants = get_max_metric_and_variant_counts(
-            query=query, **search_args,
+            query=query,
+            **search_args,
         )
         max_variants = int(max_variants // last_iterations_per_plot)
 
@@ -626,8 +659,10 @@ class EventBLL(object):
         Return events and next scroll id from the scrolled query
         Release the scroll once it is exhausted
         """
-        total_events = safe_get(es_res, "hits/total/value", default=0)
-        events = [doc["_source"] for doc in safe_get(es_res, "hits/hits", default=[])]
+        total_events = nested_get(es_res, ("hits", "total", "value"), default=0)
+        events = [
+            doc["_source"] for doc in nested_get(es_res, ("hits", "hits"), default=[])
+        ]
         next_scroll_id = es_res.get("_scroll_id")
         if next_scroll_id and not events:
             self.clear_scroll(next_scroll_id)
@@ -636,9 +671,11 @@ class EventBLL(object):
         return events, total_events, next_scroll_id
 
     def get_debug_image_urls(
-        self, company_id: str, task_id: str, after_key: dict = None
+        self, company_id: str, task_ids: Sequence[str], after_key: dict = None
     ) -> Tuple[Sequence[str], Optional[dict]]:
-        if check_empty_data(self.es, company_id, EventType.metrics_image):
+        if not task_ids or check_empty_data(
+            self.es, company_id, EventType.metrics_image
+        ):
             return [], None
 
         es_req = {
@@ -654,7 +691,10 @@ class EventBLL(object):
             },
             "query": {
                 "bool": {
-                    "must": [{"term": {"task": task_id}}, {"exists": {"field": "url"}}]
+                    "must": [
+                        {"terms": {"task": task_ids}},
+                        {"exists": {"field": "url"}},
+                    ]
                 }
             },
         }
@@ -672,9 +712,13 @@ class EventBLL(object):
         return [bucket["key"]["url"] for bucket in res["buckets"]], res.get("after_key")
 
     def get_plot_image_urls(
-        self, company_id: str, task_id: str, scroll_id: Optional[str]
+        self, company_id: str, task_ids: Sequence[str], scroll_id: Optional[str]
     ) -> Tuple[Sequence[dict], Optional[str]]:
-        if scroll_id == self.empty_scroll:
+        if (
+            scroll_id == self.empty_scroll
+            or not task_ids
+            or check_empty_data(self.es, company_id, EventType.metrics_plot)
+        ):
             return [], None
 
         if scroll_id:
@@ -689,7 +733,7 @@ class EventBLL(object):
                 "query": {
                     "bool": {
                         "must": [
-                            {"term": {"task": task_id}},
+                            {"terms": {"task": task_ids}},
                             {"exists": {"field": PlotFields.source_urls}},
                         ]
                     }
@@ -825,7 +869,8 @@ class EventBLL(object):
         query = {"bool": {"must": [{"term": {"task": task_id}}]}}
         search_args = dict(es=self.es, company_id=company_id, event_type=event_type)
         max_metrics, max_variants = get_max_metric_and_variant_counts(
-            query=query, **search_args,
+            query=query,
+            **search_args,
         )
         es_req = {
             "size": 0,
@@ -879,7 +924,8 @@ class EventBLL(object):
         }
         search_args = dict(es=self.es, company_id=company_id, event_type=event_type)
         max_metrics, max_variants = get_max_metric_and_variant_counts(
-            query=query, **search_args,
+            query=query,
+            **search_args,
         )
         max_variants = int(max_variants // 2)
         es_req = {
@@ -1023,9 +1069,9 @@ class EventBLL(object):
                                         "order": {"_key": "desc"},
                                     }
                                 }
-                            }
+                            },
                         }
-                    }
+                    },
                 }
             },
             "query": {"bool": {"must": must}},
@@ -1091,7 +1137,10 @@ class EventBLL(object):
 
         with translate_errors_context():
             es_res = search_company_events(
-                self.es, company_id=company_ids, event_type=event_type, body=es_req,
+                self.es,
+                company_id=company_ids,
+                event_type=event_type,
+                body=es_req,
             )
 
         if "aggregations" not in es_res:
@@ -1102,34 +1151,6 @@ class EventBLL(object):
             for tb in es_res["aggregations"]["tasks"]["buckets"]
         }
 
-    @staticmethod
-    def _validate_model_state(
-        company_id: str, model_id: str, allow_locked: bool = False
-    ):
-        extra_msg = None
-        query = Q(id=model_id, company=company_id)
-        if not allow_locked:
-            query &= Q(ready__ne=True)
-            extra_msg = "or model published"
-        res = Model.objects(query).only("id").first()
-        if not res:
-            raise errors.bad_request.InvalidModelId(
-                extra_msg, company=company_id, id=model_id
-            )
-
-    @staticmethod
-    def _validate_task_state(company_id: str, task_id: str, allow_locked: bool = False):
-        extra_msg = None
-        query = Q(id=task_id, company=company_id)
-        if not allow_locked:
-            query &= Q(status__nin=LOCKED_TASK_STATUSES)
-            extra_msg = "or task published"
-        res = Task.objects(query).only("id").first()
-        if not res:
-            raise errors.bad_request.InvalidTaskId(
-                extra_msg, company=company_id, id=task_id
-            )
-
     @staticmethod
     def _get_events_deletion_params(async_delete: bool) -> dict:
         if async_delete:
@@ -1143,40 +1164,52 @@ class EventBLL(object):
         return {"refresh": True}
 
     def delete_task_events(
-        self, company_id, task_id, allow_locked=False, model=False, async_delete=False,
+        self,
+        company_id,
+        task_ids: Union[str, Sequence[str]],
+        wait_for_delete: bool,
+        model=False,
     ):
-        if model:
-            self._validate_model_state(
-                company_id=company_id, model_id=task_id, allow_locked=allow_locked,
-            )
-        else:
-            self._validate_task_state(
-                company_id=company_id, task_id=task_id, allow_locked=allow_locked
-            )
-
-        es_req = {"query": {"term": {"task": task_id}}}
+        """
+        Delete task events. No check is done for tasks write access
+        so it should be checked by the calling code
+        """
+        if isinstance(task_ids, str):
+            task_ids = [task_ids]
+        deleted = 0
         with translate_errors_context():
-            es_res = delete_company_events(
-                es=self.es,
-                company_id=company_id,
-                event_type=EventType.all,
-                body=es_req,
-                **self._get_events_deletion_params(async_delete),
-            )
+            async_delete = async_task_events_delete and not wait_for_delete
+            if async_delete and len(task_ids) < 100:
+                total = self.events_iterator.count_task_events(
+                    event_type=EventType.all,
+                    company_id=company_id,
+                    task_ids=task_ids,
+                )
+                if total <= async_delete_threshold:
+                    async_delete = False
+            for tasks in chunked_iter(task_ids, 100):
+                es_req = {"query": {"terms": {"task": tasks}}}
+                es_res = delete_company_events(
+                    es=self.es,
+                    company_id=company_id,
+                    event_type=EventType.all,
+                    body=es_req,
+                    **self._get_events_deletion_params(async_delete),
+                )
+                if not async_delete:
+                    deleted += es_res.get("deleted", 0)
 
         if not async_delete:
-            return es_res.get("deleted", 0)
+            return deleted
 
     def clear_task_log(
         self,
         company_id: str,
         task_id: str,
-        allow_locked: bool = False,
         threshold_sec: int = None,
+        include_metrics: Sequence[str] = None,
+        exclude_metrics: Sequence[str] = None,
     ):
-        self._validate_task_state(
-            company_id=company_id, task_id=task_id, allow_locked=allow_locked
-        )
         if check_empty_data(
             self.es, company_id=company_id, event_type=EventType.task_log
         ):
@@ -1197,8 +1230,16 @@ class EventBLL(object):
                     }
                 )
                 sort = {"timestamp": {"order": "desc"}}
+
+            if include_metrics:
+                must.append({"terms": {"metric": include_metrics}})
+
+            more_conditions = {}
+            if exclude_metrics:
+                more_conditions = {"must_not": [{"terms": {"metric": exclude_metrics}}]}
+
             es_req = {
-                "query": {"bool": {"must": must}},
+                "query": {"bool": {"must": must, **more_conditions}},
                 **({"sort": sort} if sort else {}),
             }
             es_res = delete_company_events(
@@ -1210,30 +1251,6 @@ class EventBLL(object):
             )
             return es_res.get("deleted", 0)
 
-    def delete_multi_task_events(
-        self, company_id: str, task_ids: Sequence[str], async_delete=False
-    ):
-        """
-        Delete mutliple task events. No check is done for tasks write access
-        so it should be checked by the calling code
-        """
-        deleted = 0
-        with translate_errors_context():
-            for tasks in chunked_iter(task_ids, 100):
-                es_req = {"query": {"terms": {"task": tasks}}}
-                es_res = delete_company_events(
-                    es=self.es,
-                    company_id=company_id,
-                    event_type=EventType.all,
-                    body=es_req,
-                    **self._get_events_deletion_params(async_delete),
-                )
-                if not async_delete:
-                    deleted += es_res.get("deleted", 0)
-
-        if not async_delete:
-            return es_res.get("deleted", 0)
-
     def clear_scroll(self, scroll_id: str):
         if scroll_id == self.empty_scroll:
             return

apiserver/bll/event/event_common.py

@@ -9,7 +9,7 @@ from elasticsearch import Elasticsearch
 from apiserver.config_repo import config
 from apiserver.database.errors import translate_errors_context
 from apiserver.database.model.task.task import Task
-from apiserver.tools import safe_get
+from apiserver.utilities.dicts import nested_get
 
 
 class EventType(Enum):
@@ -123,8 +123,8 @@ def get_max_metric_and_variant_counts(
             es, company_id=company_id, event_type=event_type, body=es_req, **kwargs,
         )
 
-    metrics_count = safe_get(
-        es_res, "aggregations/metrics_count/value", max_metrics_count
+    metrics_count = nested_get(
+        es_res, ("aggregations", "metrics_count", "value"), max_metrics_count
     )
     if not metrics_count:
         return max_metrics_count, max_variants_count

apiserver/bll/event/event_metrics.py

@@ -21,9 +21,10 @@ from apiserver.bll.event.event_common import (
     TaskCompanies,
 )
 from apiserver.bll.event.scalar_key import ScalarKey, ScalarKeyEnum
+from apiserver.bll.query import Builder as QueryBuilder
 from apiserver.config_repo import config
 from apiserver.database.errors import translate_errors_context
-from apiserver.tools import safe_get
+from apiserver.utilities.dicts import nested_get
 
 log = config.logger(__file__)
 
@@ -161,7 +162,9 @@ class EventMetrics:
         return res
 
     def get_task_single_value_metrics(
-        self, companies: TaskCompanies
+        self,
+        companies: TaskCompanies,
+        metric_variants: MetricVariants = None,
     ) -> Mapping[str, dict]:
         """
         For the requested tasks return all the events delivered for the single iteration (-2**31)
@@ -179,7 +182,13 @@ class EventMetrics:
         with ThreadPoolExecutor(max_workers=EventSettings.max_workers) as pool:
             task_events = list(
                 itertools.chain.from_iterable(
-                    pool.map(self._get_task_single_value_metrics, companies.items())
+                    pool.map(
+                        partial(
+                            self._get_task_single_value_metrics,
+                            metric_variants=metric_variants,
+                        ),
+                        companies.items(),
+                    )
                 ),
             )
 
@@ -195,19 +204,19 @@ class EventMetrics:
         }
 
     def _get_task_single_value_metrics(
-        self, tasks: Tuple[str, Sequence[str]]
+        self, tasks: Tuple[str, Sequence[str]], metric_variants: MetricVariants = None
     ) -> Sequence[dict]:
         company_id, task_ids = tasks
+        must = [
+            {"terms": {"task": task_ids}},
+            {"term": {"iter": SINGLE_SCALAR_ITERATION}},
+        ]
+        if metric_variants:
+            must.append(get_metric_variants_condition(metric_variants))
+
         es_req = {
             "size": 10000,
-            "query": {
-                "bool": {
-                    "must": [
-                        {"terms": {"task": task_ids}},
-                        {"term": {"iter": SINGLE_SCALAR_ITERATION}},
-                    ]
-                }
-            },
+            "query": {"bool": {"must": must}},
         }
         with translate_errors_context():
             es_res = search_company_events(
@@ -280,7 +289,8 @@ class EventMetrics:
         query = {"bool": {"must": must}}
         search_args = dict(es=self.es, company_id=company_id, event_type=event_type)
         max_metrics, max_variants = get_max_metric_and_variant_counts(
-            query=query, **search_args,
+            query=query,
+            **search_args,
         )
         max_variants = int(max_variants // 2)
         es_req = {
@@ -332,12 +342,12 @@ class EventMetrics:
         total amount of intervals does not exceeds the samples
         Return the interval and resulting amount of intervals
         """
-        count = safe_get(data, "count/value", default=0)
+        count = nested_get(data, ("count", "value"), default=0)
         if count < samples:
             return metric, variant, 1, count
 
-        min_index = safe_get(data, "min_index/value", default=0)
-        max_index = safe_get(data, "max_index/value", default=min_index)
+        min_index = nested_get(data, ("min_index", "value"), default=0)
+        max_index = nested_get(data, ("max_index", "value"), default=min_index)
         index_range = max_index - min_index + 1
         interval = max(1, math.ceil(float(index_range) / samples))
         max_samples = math.ceil(float(index_range) / interval)
@@ -366,7 +376,8 @@ class EventMetrics:
         query = self._get_task_metrics_query(task_id=task_id, metrics=metrics)
         search_args = dict(es=self.es, company_id=company_id, event_type=event_type)
         max_metrics, max_variants = get_max_metric_and_variant_counts(
-            query=query, **search_args,
+            query=query,
+            **search_args,
         )
         max_variants = int(max_variants // 2)
         es_req = {
@@ -432,7 +443,9 @@ class EventMetrics:
 
     @classmethod
     def _get_task_metrics_query(
-        cls, task_id: str, metrics: Sequence[Tuple[str, str]],
+        cls,
+        task_id: str,
+        metrics: Sequence[Tuple[str, str]],
     ):
         must = cls._task_conditions(task_id)
         if metrics:
@@ -451,12 +464,96 @@ class EventMetrics:
 
         return {"bool": {"must": must}}
 
+    def get_multi_task_metrics(self, companies: TaskCompanies, event_type: EventType) -> Mapping[str, list]:
+        """
+        For the requested tasks return reported metrics and variants
+        """
+        tasks_ids = {
+            company: [t.id for t in tasks]
+            for company, tasks in companies.items()
+        }
+        with ThreadPoolExecutor(EventSettings.max_workers) as pool:
+            companies_res: Sequence = list(
+                pool.map(
+                    partial(
+                        self._get_multi_task_metrics,
+                        event_type=event_type,
+                    ),
+                    tasks_ids.items(),
+                )
+            )
+
+        if len(companies_res) == 1:
+            return companies_res[0]
+
+        res = defaultdict(set)
+        for c_res in companies_res:
+            for m, vars_ in c_res.items():
+                res[m].update(vars_)
+
+        return {
+            k: list(v)
+            for k, v in res.items()
+        }
+
+    def _get_multi_task_metrics(
+        self, company_tasks: Tuple[str, Sequence[str]], event_type: EventType
+    ) -> Mapping[str, list]:
+        company_id, task_ids = company_tasks
+        if check_empty_data(self.es, company_id, event_type):
+            return {}
+
+        search_args = dict(
+            es=self.es,
+            company_id=company_id,
+            event_type=event_type,
+        )
+        query = QueryBuilder.terms("task", task_ids)
+        max_metrics, max_variants = get_max_metric_and_variant_counts(
+            query=query,
+            **search_args,
+        )
+        es_req = {
+            "size": 0,
+            "query": query,
+            "aggs": {
+                "metrics": {
+                    "terms": {
+                        "field": "metric",
+                        "size": max_metrics,
+                        "order": {"_key": "asc"},
+                    },
+                    "aggs": {
+                        "variants": {
+                            "terms": {
+                                "field": "variant",
+                                "size": max_variants,
+                                "order": {"_key": "asc"},
+                            },
+                        }
+                    }
+                }
+            },
+        }
+
+        es_res = search_company_events(
+            body=es_req,
+            **search_args,
+        )
+        aggs_result = es_res.get("aggregations")
+        if not aggs_result:
+            return {}
+
+        return {
+            mb["key"]: [vb["key"] for vb in mb["variants"]["buckets"]]
+            for mb in aggs_result["metrics"]["buckets"]
+        }
+
     def get_task_metrics(
         self, company_id, task_ids: Sequence, event_type: EventType
     ) -> Sequence:
         """
-        For the requested tasks return all the metrics that
-        reported events of the requested types
+        For the requested tasks return reported metrics per task
         """
         if check_empty_data(self.es, company_id, event_type):
             return {}
@@ -495,5 +592,5 @@ class EventMetrics:
 
         return [
             metric["key"]
-            for metric in safe_get(es_res, "aggregations/metrics/buckets", default=[])
+            for metric in nested_get(es_res, ("aggregations", "metrics", "buckets"), default=[])
         ]

apiserver/bll/event/events_iterator.py

@@ -64,13 +64,13 @@ class EventsIterator:
         self,
         event_type: EventType,
         company_id: str,
-        task_id: str,
+        task_ids: Sequence[str],
         metric_variants: MetricVariants = None,
     ) -> int:
         if check_empty_data(self.es, company_id, event_type):
             return 0
 
-        query, _ = self._get_initial_query_and_must(task_id, metric_variants)
+        query, _ = self._get_initial_query_and_must(task_ids, metric_variants)
         es_req = {
             "query": query,
         }
@@ -100,7 +100,7 @@ class EventsIterator:
         For the last key-field value all the events are brought (even if the resulting size exceeds batch_size)
         so that events with this value will not be lost between the calls.
         """
-        query, must = self._get_initial_query_and_must(task_id, metric_variants)
+        query, must = self._get_initial_query_and_must([task_id], metric_variants)
 
         # retrieve the next batch of events
         es_req = {
@@ -158,14 +158,14 @@ class EventsIterator:
 
     @staticmethod
     def _get_initial_query_and_must(
-        task_id: str, metric_variants: MetricVariants = None
+        task_ids: Sequence[str], metric_variants: MetricVariants = None
     ) -> Tuple[dict, list]:
         if not metric_variants:
-            must = [{"term": {"task": task_id}}]
-            query = {"term": {"task": task_id}}
+            query = {"terms": {"task": task_ids}}
+            must = [query]
         else:
             must = [
-                {"term": {"task": task_id}},
+                {"terms": {"task": task_ids}},
                 get_metric_variants_condition(metric_variants),
             ]
             query = {"bool": {"must": must}}

apiserver/bll/event/history_debug_image_iterator.py

@@ -183,7 +183,7 @@ class HistoryDebugImageIterator:
         order = "desc" if navigate_earlier else "asc"
         es_req = {
             "size": 1,
-            "sort": [{"metric": order}, {"variant": order}],
+            "sort": [{"metric": order}, {"variant": order}, {"url": "desc"}],
             "query": {"bool": {"must": must_conditions}},
         }
 
@@ -242,7 +242,7 @@ class HistoryDebugImageIterator:
         ]
         es_req = {
             "size": 1,
-            "sort": [{"iter": order}, {"metric": order}, {"variant": order}],
+            "sort": [{"iter": order}, {"metric": order}, {"variant": order}, {"url": "desc"}],
             "query": {"bool": {"must": must_conditions}},
         }
         es_res = search_company_events(
@@ -338,7 +338,7 @@ class HistoryDebugImageIterator:
 
             es_req = {
                 "size": 1,
-                "sort": {"iter": "desc"},
+                "sort": [{"iter": "desc"}, {"url": "desc"}],
                 "query": {"bool": {"must": must_conditions}},
             }
 

apiserver/bll/event/metric_events_iterator.py

@@ -6,7 +6,6 @@ from operator import itemgetter
 from typing import Sequence, Tuple, Optional, Mapping, Callable
 
 import attr
-import dpath
 from boltons.iterutils import first
 from elasticsearch import Elasticsearch
 from jsonmodels.fields import StringField, ListField, IntField
@@ -27,6 +26,7 @@ from apiserver.config_repo import config
 from apiserver.database.errors import translate_errors_context
 from apiserver.database.model.task.metrics import MetricEventStats
 from apiserver.database.model.task.task import Task
+from apiserver.utilities.dicts import nested_get
 
 
 class VariantState(Base):
@@ -305,13 +305,13 @@ class MetricEventsIterator:
         return [
             MetricState(
                 metric=metric["key"],
-                timestamp=dpath.get(metric, "last_event_timestamp/value"),
+                timestamp=nested_get(metric, ("last_event_timestamp", "value")),
                 variants=[
                     init_variant_state(variant)
-                    for variant in dpath.get(metric, "variants/buckets")
+                    for variant in nested_get(metric, ("variants", "buckets"))
                 ],
             )
-            for metric in dpath.get(es_res, "aggregations/metrics/buckets")
+            for metric in nested_get(es_res, ("aggregations", "metrics", "buckets"))
         ]
 
     @abc.abstractmethod
@@ -384,7 +384,8 @@ class MetricEventsIterator:
                                     "aggs": {
                                         "events": {
                                             "top_hits": {
-                                                "sort": self._get_same_variant_events_order()
+                                                "sort": self._get_same_variant_events_order(),
+                                                "size": 1,
                                             }
                                         }
                                     },
@@ -430,14 +431,14 @@ class MetricEventsIterator:
         def get_iteration_events(it_: dict) -> Sequence:
             return [
                 self._process_event(ev["_source"])
-                for m in dpath.get(it_, "metrics/buckets")
-                for v in dpath.get(m, "variants/buckets")
-                for ev in dpath.get(v, "events/hits/hits")
+                for m in nested_get(it_, ("metrics", "buckets"))
+                for v in nested_get(m, ("variants", "buckets"))
+                for ev in nested_get(v, ("events", "hits", "hits"))
                 if is_valid_event(ev["_source"])
             ]
 
         iterations = []
-        for it in dpath.get(es_res, "aggregations/iters/buckets"):
+        for it in nested_get(es_res, ("aggregations", "iters", "buckets")):
             events = get_iteration_events(it)
             if events:
                 iterations.append({"iter": it["key"], "events": events})

apiserver/bll/model/__init__.py

@@ -6,16 +6,14 @@ from mongoengine import Q
 from apiserver.apierrors import errors
 from apiserver.apimodels.models import ModelTaskPublishResponse
 from apiserver.bll.task.utils import deleted_prefix, get_last_metric_updates
-from apiserver.config_repo import config
 from apiserver.database.model import EntityVisibility
 from apiserver.database.model.model import Model
 from apiserver.database.model.task.task import Task, TaskStatus
+from apiserver.service_repo.auth import Identity
 from .metadata import Metadata
 
 
 class ModelBLL:
-    event_bll = None
-
     @classmethod
     def get_company_model_by_id(
         cls, company_id: str, model_id: str, only_fields=None
@@ -57,14 +55,15 @@ class ModelBLL:
         cls,
         model_id: str,
         company_id: str,
-        user_id: str,
+        identity: Identity,
         force_publish_task: bool = False,
-        publish_task_func: Callable[[str, str, str, bool], dict] = None,
+        publish_task_func: Callable[[str, str, Identity, bool], dict] = None,
     ) -> Tuple[int, ModelTaskPublishResponse]:
         model = cls.get_company_model_by_id(company_id=company_id, model_id=model_id)
         if model.ready:
             raise errors.bad_request.ModelIsReady(company=company_id, model=model_id)
 
+        user_id = identity.user
         published_task = None
         if model.task and publish_task_func:
             task = (
@@ -74,18 +73,25 @@ class ModelBLL:
             )
             if task and task.status != TaskStatus.published:
                 task_publish_res = publish_task_func(
-                    model.task, company_id, user_id, force_publish_task
+                    model.task, company_id, identity, force_publish_task
                 )
                 published_task = ModelTaskPublishResponse(
                     id=model.task, data=task_publish_res
                 )
 
-        updated = model.update(upsert=False, ready=True, last_update=datetime.utcnow())
+        now = datetime.utcnow()
+        updated = model.update(
+            upsert=False,
+            ready=True,
+            last_update=now,
+            last_change=now,
+            last_changed_by=user_id,
+        )
         return updated, published_task
 
     @classmethod
     def delete_model(
-        cls, model_id: str, company_id: str, user_id: str, force: bool, delete_external_artifacts: bool = True,
+        cls, model_id: str, company_id: str, user_id: str, force: bool
     ) -> Tuple[int, Model]:
         model = cls.get_company_model_by_id(
             company_id=company_id,
@@ -125,6 +131,7 @@ class ModelBLL:
                                 "models.output.$[elem].model": deleted_model_id,
                                 "output.error": f"model deleted on {now.isoformat()}",
                                 "last_change": now,
+                                "last_changed_by": user_id,
                             },
                         },
                         array_filters=[{"elem.model": model_id}],
@@ -132,60 +139,38 @@ class ModelBLL:
                     )
                 else:
                     task.update(
-                        pull__models__output__model=model_id, set__last_change=now
+                        pull__models__output__model=model_id,
+                        set__last_change=now,
+                        set__last_changed_by=user_id,
                     )
 
-        delete_external_artifacts = delete_external_artifacts and config.get(
-            "services.async_urls_delete.enabled", True
-        )
-        if delete_external_artifacts:
-            from apiserver.bll.task.task_cleanup import (
-                collect_debug_image_urls,
-                collect_plot_image_urls,
-                _schedule_for_delete,
-            )
-            urls = set()
-            urls.update(collect_debug_image_urls(company_id, model_id))
-            urls.update(collect_plot_image_urls(company_id, model_id))
-            if model.uri:
-                urls.add(model.uri)
-            if urls:
-                _schedule_for_delete(
-                    task_id=model_id,
-                    company=company_id,
-                    user=user_id,
-                    urls=urls,
-                    can_delete_folders=False,
-                )
-
-        if not cls.event_bll:
-            from apiserver.bll.event import EventBLL
-            cls.event_bll = EventBLL()
-
-        cls.event_bll.delete_task_events(company_id, model_id, allow_locked=True, model=True)
         del_count = Model.objects(id=model_id, company=company_id).delete()
         return del_count, model
 
     @classmethod
-    def archive_model(cls, model_id: str, company_id: str):
+    def archive_model(cls, model_id: str, company_id: str, user_id: str):
         cls.get_company_model_by_id(
             company_id=company_id, model_id=model_id, only_fields=("id",)
         )
+        now = datetime.utcnow()
         archived = Model.objects(company=company_id, id=model_id).update(
             add_to_set__system_tags=EntityVisibility.archived.value,
-            last_update=datetime.utcnow(),
+            last_change=now,
+            last_changed_by=user_id,
         )
 
         return archived
 
     @classmethod
-    def unarchive_model(cls, model_id: str, company_id: str):
+    def unarchive_model(cls, model_id: str, company_id: str, user_id: str):
         cls.get_company_model_by_id(
             company_id=company_id, model_id=model_id, only_fields=("id",)
         )
+        now = datetime.utcnow()
         unarchived = Model.objects(company=company_id, id=model_id).update(
             pull__system_tags=EntityVisibility.archived.value,
-            last_update=datetime.utcnow(),
+            last_change=now,
+            last_changed_by=user_id,
         )
 
         return unarchived
@@ -201,7 +186,7 @@ class ModelBLL:
             [
                 {
                     "$match": {
-                        "company": {"$in": [None, "", company]},
+                        "company": {"$in": ["", company]},
                         "_id": {"$in": model_ids},
                     }
                 },
@@ -218,11 +203,18 @@ class ModelBLL:
     @staticmethod
     def update_statistics(
         company_id: str,
+        user_id: str,
         model_id: str,
+        last_update: datetime = None,
         last_iteration_max: int = None,
         last_scalar_events: Dict[str, Dict[str, dict]] = None,
     ):
-        updates = {"last_update": datetime.utcnow()}
+        last_update = last_update or datetime.utcnow()
+        updates = {
+            "last_update": datetime.utcnow(),
+            "last_change": last_update,
+            "last_changed_by": user_id,
+        }
         if last_iteration_max is not None:
             updates.update(max__last_iteration=last_iteration_max)
 

apiserver/bll/organization/__init__.py

@@ -1,8 +1,12 @@
 from collections import defaultdict
+from datetime import datetime
 from enum import Enum
-from typing import Sequence, Dict
+from typing import Sequence, Dict, Type
 
+from apiserver.apierrors import errors
+from apiserver.bll.util import update_project_time
 from apiserver.config_repo import config
+from apiserver.database.model.model import AttributedDocument
 from apiserver.database.model.model import Model
 from apiserver.database.model.task.task import Task
 from apiserver.redis_manager import redman
@@ -22,6 +26,56 @@ class OrgBLL:
         self._task_tags = _TagsCache(Task, self.redis)
         self._model_tags = _TagsCache(Model, self.redis)
 
+    def edit_entity_tags(
+        self,
+        company_id,
+        user_id: str,
+        entity_cls: Type[AttributedDocument],
+        entity_ids: Sequence[str],
+        add_tags: Sequence[str],
+        remove_tags: Sequence[str],
+    ) -> int:
+        if entity_cls not in (Task, Model):
+            raise errors.bad_request.ValidationError(
+                "Tags editing can be called on tasks or models only"
+            )
+        if not entity_ids:
+            raise errors.bad_request.ValidationError(
+                "No entity ids provided for editing tags"
+            )
+        if not (add_tags or remove_tags):
+            raise errors.bad_request.ValidationError(
+                "Either add tags or remove tags should be provided"
+            )
+
+        updated = 0
+        last_changed = {
+            "set__last_change": datetime.utcnow(),
+            "set__last_changed_by": user_id,
+        }
+        if add_tags:
+            updated += entity_cls.objects(company=company_id, id__in=entity_ids).update(
+                add_to_set__tags=add_tags, **last_changed,
+            )
+        if remove_tags:
+            updated += entity_cls.objects(company=company_id, id__in=entity_ids).update(
+                pull_all__tags=remove_tags, **last_changed,
+            )
+        if not updated:
+            return 0
+
+        projects = entity_cls.objects(company=company_id, id__in=entity_ids).distinct(
+            "project"
+        )
+        update_project_time(project_ids=projects)
+        self.update_tags(
+            company_id,
+            entity=Tags.Task if entity_cls is Task else Tags.Model,
+            projects=projects,
+            tags=add_tags or remove_tags
+        )
+        return updated
+
     def get_tags(
         self,
         company_id: str,
@@ -50,10 +104,10 @@ class OrgBLL:
         return ret
 
     def update_tags(
-        self, company_id: str, entity: Tags, project: str, tags=None, system_tags=None,
+        self, company_id: str, entity: Tags, projects: Sequence[str], tags=None, system_tags=None,
     ):
         tags_cache = self._get_tags_cache_for_entity(entity)
-        tags_cache.update_tags(company_id, project, tags, system_tags)
+        tags_cache.update_tags(company_id, projects, tags, system_tags)
 
     def reset_tags(self, company_id: str, entity: Tags, projects: Sequence[str]):
         tags_cache = self._get_tags_cache_for_entity(entity)

apiserver/bll/organization/tags_cache.py

@@ -6,7 +6,6 @@ from redis import Redis
 
 from apiserver.config_repo import config
 from apiserver.bll.project import project_ids_with_children
-from apiserver.database.model import EntityVisibility
 from apiserver.database.model.base import GetMixin
 from apiserver.database.model.model import Model
 from apiserver.database.model.task.task import Task
@@ -43,8 +42,8 @@ class _TagsCache:
                     query &= GetMixin.get_list_field_query(name, vals)
         if project:
             query &= Q(project__in=project_ids_with_children([project]))
-        else:
-            query &= Q(system_tags__nin=[EntityVisibility.hidden.value])
+        # else:
+        #     query &= Q(system_tags__nin=[EntityVisibility.hidden.value])
 
         return self.db_cls.objects(query).distinct(field)
 
@@ -107,7 +106,7 @@ class _TagsCache:
 
         return ret
 
-    def update_tags(self, company_id: str, project: str, tags=None, system_tags=None):
+    def update_tags(self, company_id: str, projects: Sequence[str], tags=None, system_tags=None):
         """
         Updates tags. If reset is set then both tags and system_tags
         are recalculated. Otherwise only those that are not 'None'
@@ -123,7 +122,7 @@ class _TagsCache:
         if not fields:
             return
 
-        self._delete_redis_keys(company_id, projects=[project], fields=fields)
+        self._delete_redis_keys(company_id, projects=projects, fields=fields)
 
     def reset_tags(self, company_id: str, projects: Sequence[str]):
         self._delete_redis_keys(

apiserver/bll/project/project_bll.py

@@ -41,6 +41,7 @@ from .sub_projects import (
     _ids_with_parents,
     _get_project_depth,
     ProjectsChildren,
+    _get_writable_project_from_name,
 )
 
 log = config.logger(__file__)
@@ -225,6 +226,18 @@ class ProjectBLL:
             raise errors.bad_request.ProjectPathExceedsMax(max_depth=max_depth)
 
         name, location = _validate_project_name(name)
+
+        existing = _get_writable_project_from_name(
+            company=company,
+            name=name,
+        )
+        if existing:
+            raise errors.bad_request.ExpectedUniqueData(
+                replacement_msg="Project with the same name already exists",
+                name=name,
+                company=company,
+            )
+
         now = datetime.utcnow()
         project = Project(
             id=database.utils.id(),
@@ -315,11 +328,12 @@ class ProjectBLL:
                 description="",
             )
 
-        extra = (
-            {"set__last_change": datetime.utcnow()}
-            if hasattr(entity_cls, "last_change")
-            else {}
-        )
+        extra = {}
+        if hasattr(entity_cls, "last_change"):
+            extra["set__last_change"] = datetime.utcnow()
+        if hasattr(entity_cls, "last_changed_by"):
+            extra["set__last_changed_by"] = user
+
         entity_cls.objects(company=company, id__in=ids).update(
             set__project=project, **extra
         )
@@ -340,6 +354,17 @@ class ProjectBLL:
     ) -> Tuple[Sequence, Sequence]:
         archived = EntityVisibility.archived.value
 
+        def project_task_fields():
+            return {
+                "$project": {
+                    "project": 1,
+                    "status": 1,
+                    "system_tags": 1,
+                    "started": 1,
+                    "completed": 1,
+                }
+            }
+
         def ensure_valid_fields():
             """
             Make sure system tags is always an array (required by subsequent $in in archived_tasks_cond
@@ -367,6 +392,7 @@ class ProjectBLL:
                     users=users,
                 )
             },
+            project_task_fields(),
             ensure_valid_fields(),
             {
                 "$group": {
@@ -515,6 +541,7 @@ class ProjectBLL:
                     users=users,
                 )
             },
+            project_task_fields(),
             ensure_valid_fields(),
             {
                 # for each project
@@ -550,7 +577,10 @@ class ProjectBLL:
 
     @classmethod
     def get_dataset_stats(
-        cls, company: str, project_ids: Sequence[str], users: Sequence[str] = None,
+        cls,
+        company: str,
+        project_ids: Sequence[str],
+        users: Sequence[str] = None,
     ) -> Dict[str, dict]:
         if not project_ids:
             return {}
@@ -584,7 +614,9 @@ class ProjectBLL:
 
     @staticmethod
     def _get_projects_children(
-        project_ids: Sequence[str], search_hidden: bool, allowed_ids: Sequence[str],
+        project_ids: Sequence[str],
+        search_hidden: bool,
+        allowed_ids: Sequence[str],
     ) -> Tuple[ProjectsChildren, Set[str]]:
         child_projects = _get_sub_projects(
             project_ids,
@@ -628,7 +660,9 @@ class ProjectBLL:
         project_ids_with_children = set(project_ids)
         if include_children:
             child_projects, children_ids = cls._get_projects_children(
-                project_ids, search_hidden=True, allowed_ids=selected_project_ids,
+                project_ids,
+                search_hidden=True,
+                allowed_ids=selected_project_ids,
             )
             project_ids_with_children |= children_ids
 
@@ -848,7 +882,7 @@ class ProjectBLL:
         company,
         project_ids: Sequence[str],
         user_ids: Optional[Sequence[str]] = None,
-    ) -> Set[str]:
+    ) -> Set[Union[str, type(None)]]:
         """
         Get the set of user ids that created tasks/models in the given projects
         If project_ids is empty then all projects are examined
@@ -902,6 +936,7 @@ class ProjectBLL:
         allow_public: bool = True,
         children_type: ProjectChildrenType = None,
         children_tags: Sequence[str] = None,
+        children_tags_filter: dict = None,
     ) -> Tuple[Sequence[str], Sequence[str]]:
         """
         Get the projects ids matching children_condition (if passed) or where the passed user created any tasks
@@ -922,11 +957,15 @@ class ProjectBLL:
             query &= Q(user__in=users)
 
         project_query = None
-        child_query = (
-            query & GetMixin.get_list_field_query("tags", children_tags)
-            if children_tags
-            else query
-        )
+        if children_tags_filter:
+            child_query = query & GetMixin.get_list_filter_query(
+                "tags", children_tags_filter
+            )
+        elif children_tags:
+            child_query = query & GetMixin.get_list_field_query("tags", children_tags)
+        else:
+            child_query = query
+
         if children_type == ProjectChildrenType.dataset:
             child_queries = {
                 Project: child_query
@@ -989,8 +1028,8 @@ class ProjectBLL:
             if include_subprojects:
                 projects = _ids_with_children(projects)
             query &= Q(project__in=projects)
-        else:
-            query &= Q(system_tags__nin=[EntityVisibility.hidden.value])
+        # else:
+        #     query &= Q(system_tags__nin=[EntityVisibility.hidden.value])
 
         if state == EntityVisibility.archived:
             query &= Q(system_tags__in=[EntityVisibility.archived.value])
@@ -1075,7 +1114,7 @@ class ProjectBLL:
         project_field: str = "project",
     ):
         conditions = {
-            "company": {"$in": [None, "", company]},
+            "company": {"$in": ["", company]},
             project_field: {"$in": project_ids},
         }
         if users:
@@ -1086,39 +1125,50 @@ class ProjectBLL:
 
         or_conditions = []
         for field, field_filter in filter_.items():
-            if not (
-                field_filter
-                and isinstance(field_filter, list)
-                and all(isinstance(t, str) for t in field_filter)
-            ):
+            if not (field_filter and isinstance(field_filter, (list, dict))):
                 raise errors.bad_request.ValidationError(
-                    f"List of strings expected for the field: {field}"
+                    f"Non empty list or dictionary expected for the field: {field}"
                 )
-            helper = GetMixin.NewListFieldBucketHelper(
-                field, data=field_filter, legacy=True
-            )
-            field_conditions = {}
-            for action, values in helper.actions.items():
-                value = list(set(values))
-                for key in reversed(action.split("__")):
-                    value = {f"${key}": value}
-                field_conditions.update(value)
-            if (
-                helper.explicit_operator
-                and helper.global_operator == Q.OR
-                and len(field_conditions) > 1
-            ):
-                or_conditions.append(
-                    [{field: {op: cond}} for op, cond in field_conditions.items()]
+
+            if isinstance(field_filter, list):
+                if not all(isinstance(t, str) for t in field_filter):
+                    raise errors.bad_request.ValidationError(
+                        f"Only string values are allowed in the list filter: {field}"
+                    )
+                helper = GetMixin.NewListFieldBucketHelper(
+                    field, data=field_filter, legacy=True
                 )
+                op = helper.global_operator
+                db_query = {op: helper.actions}
             else:
-                conditions[field] = field_conditions
+                helper = GetMixin.ListQueryFilter.from_data(field, field_filter)
+                db_query = helper.db_query
+
+            for op, actions in db_query.items():
+                field_conditions = {}
+                for action, values in actions.items():
+                    value = list(set(values)) if isinstance(values, list) else values
+                    for key in reversed(action.split("__")):
+                        value = {f"${key}": value}
+                    field_conditions.update(value)
+
+                if op == Q.OR and len(field_conditions) > 1:
+                    or_conditions.append(
+                        {
+                            "$or": [
+                                {field: {db_modifier: cond}}
+                                for db_modifier, cond in field_conditions.items()
+                            ]
+                        }
+                    )
+                else:
+                    conditions[field] = field_conditions
 
         if or_conditions:
             if len(or_conditions) == 1:
-                conditions["$or"] = next(iter(or_conditions))
+                conditions = next(iter(or_conditions))
             else:
-                conditions["$and"] = [{"$or": c} for c in or_conditions]
+                conditions["$and"] = [c for c in or_conditions]
 
         return conditions
 

apiserver/bll/project/project_cleanup.py

@@ -8,10 +8,9 @@ from mongoengine import Q
 from apiserver.apierrors import errors
 from apiserver.bll.event import EventBLL
 from apiserver.bll.task.task_cleanup import (
-    collect_debug_image_urls,
-    collect_plot_image_urls,
     TaskUrls,
-    _schedule_for_delete,
+    schedule_for_delete,
+    delete_task_events_and_collect_urls,
 )
 from apiserver.config_repo import config
 from apiserver.database.model import EntityVisibility
@@ -30,7 +29,6 @@ from .sub_projects import _ids_with_children
 
 log = config.logger(__file__)
 event_bll = EventBLL()
-async_events_delete = config.get("services.tasks.async_events_delete", False)
 
 
 @attr.s(auto_attribs=True)
@@ -83,7 +81,8 @@ def validate_project_delete(company: str, project_id: str):
         ret["pipelines"] = 0
     if dataset_ids:
         datasets_with_data = Task.objects(
-            project__in=dataset_ids, system_tags__nin=[EntityVisibility.archived.value],
+            project__in=dataset_ids,
+            system_tags__nin=[EntityVisibility.archived.value],
         ).distinct("project")
         ret["datasets"] = len(datasets_with_data)
     else:
@@ -185,14 +184,14 @@ def delete_project(
         res = DeleteProjectResult(disassociated_tasks=disassociated[Task])
     else:
         deleted_models, model_event_urls, model_urls = _delete_models(
-            company=company, projects=project_ids
+            company=company, user=user, projects=project_ids
         )
         deleted_tasks, task_event_urls, artifact_urls = _delete_tasks(
-            company=company, projects=project_ids
+            company=company, user=user, projects=project_ids
         )
         event_urls = task_event_urls | model_event_urls
         if delete_external_artifacts:
-            scheduled = _schedule_for_delete(
+            scheduled = schedule_for_delete(
                 task_id=project_id,
                 company=company,
                 user=user,
@@ -206,7 +205,6 @@ def delete_project(
             deleted_models=deleted_models,
             urls=TaskUrls(
                 model_urls=list(model_urls),
-                event_urls=list(event_urls),
                 artifact_urls=list(artifact_urls),
             ),
         )
@@ -217,7 +215,9 @@ def delete_project(
     return res, affected
 
 
-def _delete_tasks(company: str, projects: Sequence[str]) -> Tuple[int, Set, Set]:
+def _delete_tasks(
+    company: str, user: str, projects: Sequence[str]
+) -> Tuple[int, Set, Set]:
     """
     Delete only the task themselves and their non published version.
     Child models under the same project are deleted separately.
@@ -228,14 +228,21 @@ def _delete_tasks(company: str, projects: Sequence[str]) -> Tuple[int, Set, Set]
     if not tasks:
         return 0, set(), set()
 
-    task_ids = {t.id for t in tasks}
-    Task.objects(parent__in=task_ids, project__nin=projects).update(parent=None)
-    Model.objects(task__in=task_ids, project__nin=projects).update(task=None)
+    task_ids = list({t.id for t in tasks})
+    now = datetime.utcnow()
+    Task.objects(parent__in=task_ids, project__nin=projects).update(
+        parent=None,
+        last_change=now,
+        last_changed_by=user,
+    )
+    Model.objects(task__in=task_ids, project__nin=projects).update(
+        task=None,
+        last_change=now,
+        last_changed_by=user,
+    )
 
-    event_urls, artifact_urls = set(), set()
+    artifact_urls = set()
     for task in tasks:
-        event_urls.update(collect_debug_image_urls(company, task.id))
-        event_urls.update(collect_plot_image_urls(company, task.id))
         if task.execution and task.execution.artifacts:
             artifact_urls.update(
                 {
@@ -245,15 +252,16 @@ def _delete_tasks(company: str, projects: Sequence[str]) -> Tuple[int, Set, Set]
                 }
             )
 
-    event_bll.delete_multi_task_events(
-        company, list(task_ids), async_delete=async_events_delete
+    event_urls = delete_task_events_and_collect_urls(
+        company=company, task_ids=task_ids, wait_for_delete=False
     )
     deleted = tasks.delete()
+
     return deleted, event_urls, artifact_urls
 
 
 def _delete_models(
-    company: str, projects: Sequence[str]
+    company: str, user: str, projects: Sequence[str]
 ) -> Tuple[int, Set[str], Set[str]]:
     """
     Delete project models and update the tasks from other projects
@@ -287,25 +295,30 @@ def _delete_models(
                 "status": TaskStatus.published,
             },
             update={
-                "$set": {"models.output.$[elem].model": deleted, "last_change": now,}
+                "$set": {
+                    "models.output.$[elem].model": deleted,
+                    "last_change": now,
+                    "last_changed_by": user,
+                }
             },
             array_filters=[{"elem.model": {"$in": model_ids}}],
             upsert=False,
         )
         # update unpublished tasks
         Task.objects(
-            id__in=model_tasks, project__nin=projects, status__ne=TaskStatus.published,
-        ).update(pull__models__output__model__in=model_ids, set__last_change=now)
+            id__in=model_tasks,
+            project__nin=projects,
+            status__ne=TaskStatus.published,
+        ).update(
+            pull__models__output__model__in=model_ids,
+            set__last_change=now,
+            set__last_changed_by=user,
+        )
 
-    event_urls, model_urls = set(), set()
-    for m in models:
-        event_urls.update(collect_debug_image_urls(company, m.id))
-        event_urls.update(collect_plot_image_urls(company, m.id))
-        if m.uri:
-            model_urls.add(m.uri)
-
-    event_bll.delete_multi_task_events(
-        company, model_ids, async_delete=async_events_delete
+    model_urls = {m.uri for m in models if m.uri}
+    event_urls = delete_task_events_and_collect_urls(
+        company=company, task_ids=model_ids, model=True, wait_for_delete=False
     )
     deleted = models.delete()
+
     return deleted, event_urls, model_urls

apiserver/bll/project/project_queries.py

@@ -47,7 +47,7 @@ class ProjectQueries:
     @staticmethod
     def _get_company_constraint(company_id: str, allow_public: bool = True) -> dict:
         if allow_public:
-            return {"company": {"$in": [None, "", company_id]}}
+            return {"company": {"$in": ["", company_id]}}
 
         return {"company": company_id}
 
@@ -140,6 +140,7 @@ class ProjectQueries:
         name: str,
         include_subprojects: bool,
         allow_public: bool = True,
+        pattern: str = None,
         page: int = 0,
         page_size: int = 500,
     ) -> ParamValues:
@@ -164,7 +165,20 @@ class ProjectQueries:
         if not last_updated_task:
             return 0, []
 
-        redis_key = f"hyperparam_values_{company_id}_{'_'.join(project_ids)}_{section}_{name}_{allow_public}_{page}_{page_size}"
+        redis_key = "_".join(
+            str(part)
+            for part in (
+                "hyperparam_values",
+                company_id,
+                "_".join(project_ids),
+                section,
+                name,
+                allow_public,
+                pattern,
+                page,
+                page_size,
+            )
+        )
         last_update = last_updated_task.last_update or datetime.utcnow()
         cached_res = self._get_cached_param_values(
             key=redis_key,
@@ -176,14 +190,22 @@ class ProjectQueries:
         if cached_res:
             return cached_res
 
-        pipeline = [
-            {
-                "$match": {
-                    **company_constraint,
-                    **project_constraint,
-                    key_path: {"$exists": True},
+        match_condition = {
+            **company_constraint,
+            **project_constraint,
+            key_path: {"$exists": True},
+        }
+        if pattern:
+            match_condition["$expr"] = {
+                "$regexMatch": {
+                    "input": f"${key_path}.value",
+                    "regex": pattern,
+                    "options": "i",
                 }
-            },
+            }
+
+        pipeline = [
+            {"$match": match_condition},
             {"$project": {"value": f"${key_path}.value"}},
             {"$group": {"_id": "$value"}},
             {"$sort": {"_id": 1}},
@@ -217,6 +239,7 @@ class ProjectQueries:
         company_id,
         project_ids: Sequence[str],
         include_subprojects: bool,
+        ids: Sequence[str],
         model_metrics: bool = False,
     ):
         pipeline = [
@@ -224,6 +247,7 @@ class ProjectQueries:
                 "$match": {
                     **cls._get_company_constraint(company_id),
                     **cls._get_project_constraint(project_ids, include_subprojects),
+                    **({"_id": {"$in": ids}} if ids else {}),
                 }
             },
             {"$project": {"metrics": {"$objectToArray": "$last_metrics"}}},

apiserver/bll/project/sub_projects.py

@@ -2,6 +2,8 @@ import itertools
 from datetime import datetime
 from typing import Tuple, Optional, Sequence, Mapping
 
+from boltons.iterutils import first
+
 from apiserver import database
 from apiserver.apierrors import errors
 from apiserver.database.model import EntityVisibility
@@ -96,10 +98,21 @@ def _get_writable_project_from_name(
     """
     Return a project from name. If the project not found then return None
     """
-    qs = Project.objects(company=company, name=name)
+    qs = Project.objects(company__in=[company, ""], name=name)
     if _only:
+        if "company" not in _only:
+            _only = ["company", *_only]
         qs = qs.only(*_only)
-    return qs.first()
+    projects = list(qs)
+
+    if not projects:
+        return
+
+    project = first(p for p in projects if p.company == company)
+    if not project:
+        raise errors.bad_request.PublicProjectExists(name=name)
+
+    return project
 
 
 ProjectsChildren = Mapping[str, Sequence[Project]]

apiserver/bll/query/builder.py

@@ -9,20 +9,35 @@ RANGE_IGNORE_VALUE = -1
 
 class Builder:
     @staticmethod
-    def dates_range(from_date: Union[int, float], to_date: Union[int, float]) -> dict:
+    def dates_range(
+        from_date: Optional[Union[int, float]] = None,
+        to_date: Optional[Union[int, float]] = None,
+    ) -> dict:
+        assert (
+            from_date or to_date
+        ), "range condition requires that at least one of from_date or to_date specified"
+        conditions = {}
+        if from_date:
+            conditions["gte"] = int(from_date)
+        if to_date:
+            conditions["lte"] = int(to_date)
         return {
             "range": {
                 "timestamp": {
-                    "gte": int(from_date),
-                    "lte": int(to_date),
+                    **conditions,
                     "format": "epoch_second",
                 }
             }
         }
 
     @staticmethod
-    def terms(field: str, values: Iterable[str]) -> dict:
+    def terms(field: str, values: Iterable) -> dict:
+        if isinstance(values, str):
+            assert not isinstance(values, str), "apparently 'term' should be used here"
         return {"terms": {field: list(values)}}
+    @staticmethod
+    def term(field: str, value) -> dict:
+        return {"term": {field: value}}
 
     @staticmethod
     def normalize_range(

apiserver/bll/queue/queue_bll.py

@@ -1,6 +1,6 @@
 from collections import defaultdict
 from datetime import datetime
-from typing import Sequence, Optional, Tuple, Union
+from typing import Sequence, Optional, Tuple, Union, Iterable
 
 from elasticsearch import Elasticsearch
 from mongoengine import Q
@@ -135,51 +135,78 @@ class QueueBLL(object):
             self.get_by_id(company_id=company_id, queue_id=queue_id, only=("id",))
             return Queue.safe_update(company_id, queue_id, update_fields)
 
-    def delete(self, company_id: str, user_id: str, queue_id: str, force: bool) -> None:
+    def _update_task_status_on_removal_from_queue(
+        self,
+        company_id: str,
+        user_id: str,
+        task_ids: Iterable[str],
+        queue_id: str,
+        reason: str
+    ) -> Sequence[str]:
+        from apiserver.bll.task import ChangeStatusRequest
+        tasks = []
+        for task_id in task_ids:
+            try:
+                task = Task.get(
+                    company=company_id,
+                    id=task_id,
+                    execution__queue=queue_id,
+                    _only=[
+                        "id",
+                        "company",
+                        "status",
+                        "enqueue_status",
+                        "project",
+                    ],
+                )
+                if not task:
+                    continue
+
+                tasks.append(task.id)
+                ChangeStatusRequest(
+                    task=task,
+                    new_status=task.enqueue_status or TaskStatus.created,
+                    status_reason=reason,
+                    status_message="",
+                    user_id=user_id,
+                    force=True,
+                ).execute(
+                    enqueue_status=None,
+                    unset__execution__queue=1,
+                )
+            except Exception as ex:
+                log.error(
+                    f"Failed updating task {task_id} status on removal from queue: {queue_id}, {str(ex)}"
+                )
+
+        return tasks
+
+    def delete(self, company_id: str, user_id: str, queue_id: str, force: bool) -> Sequence[str]:
         """
         Delete the queue
         :raise errors.bad_request.InvalidQueueId: if the queue is not found
         :raise errors.bad_request.QueueNotEmpty: if the queue is not empty and 'force' not set
         """
-        with translate_errors_context():
-            queue = self.get_by_id(company_id=company_id, queue_id=queue_id)
-            if queue.entries:
-                if not force:
-                    raise errors.bad_request.QueueNotEmpty(
-                        "use force=true to delete", id=queue_id
-                    )
-                from apiserver.bll.task import ChangeStatusRequest
-
-                for item in queue.entries:
-                    try:
-                        task = Task.get_for_writing(
-                            company=company_id,
-                            id=item.task,
-                            _only=[
-                                "id",
-                                "company",
-                                "status",
-                                "enqueue_status",
-                                "project",
-                            ],
-                        )
-                        if not task:
-                            continue
-
-                        ChangeStatusRequest(
-                            task=task,
-                            new_status=task.enqueue_status or TaskStatus.created,
-                            status_reason="Queue was deleted",
-                            status_message="",
-                            user_id=user_id,
-                            force=True,
-                        ).execute(enqueue_status=None)
-                    except Exception as ex:
-                        log.exception(
-                            f"Failed dequeuing task {item.task} from queue: {queue_id}"
-                        )
-
+        queue = self.get_by_id(company_id=company_id, queue_id=queue_id)
+        if not queue.entries:
             queue.delete()
+            return []
+
+        if not force:
+            raise errors.bad_request.QueueNotEmpty(
+                "use force=true to delete", id=queue_id
+            )
+
+        tasks = self._update_task_status_on_removal_from_queue(
+            company_id=company_id,
+            user_id=user_id,
+            task_ids={item.task for item in queue.entries},
+            queue_id=queue_id,
+            reason=f"Queue {queue_id} was deleted",
+        )
+
+        queue.delete()
+        return tasks
 
     def get_all(
         self,
@@ -307,7 +334,36 @@ class QueueBLL(object):
 
             return queue.entries[0]
 
-    def remove_task(self, company_id: str, queue_id: str, task_id: str) -> int:
+    def clear_queue(
+        self,
+        company_id: str,
+        user_id: str,
+        queue_id: str,
+    ):
+        queue = Queue.objects(company=company_id, id=queue_id).first()
+        if not queue:
+            raise errors.bad_request.InvalidQueueId(
+                queue=queue_id
+            )
+
+        if not queue.entries:
+            return []
+
+        tasks = self._update_task_status_on_removal_from_queue(
+            company_id=company_id,
+            user_id=user_id,
+            task_ids={item.task for item in queue.entries},
+            queue_id=queue_id,
+            reason=f"Queue {queue_id} was cleared",
+        )
+
+        queue.update(entries=[])
+        queue.reload()
+        self.metrics.log_queue_metrics_to_es(company_id=company_id, queues=[queue])
+
+        return tasks
+
+    def remove_task(self, company_id: str, user_id: str, queue_id: str, task_id: str, update_task_status: bool = False) -> int:
         """
         Removes the task from the queue and returns the number of removed items
         :raise errors.bad_request.InvalidQueueOrTaskNotQueued: if the task is not found in the queue
@@ -322,6 +378,14 @@ class QueueBLL(object):
             res = Queue.objects(entries__task=task_id, **query).update_one(
                 pull_all__entries=entries_to_remove, last_update=datetime.utcnow()
             )
+            if res and update_task_status:
+                self._update_task_status_on_removal_from_queue(
+                    company_id=company_id,
+                    user_id=user_id,
+                    task_ids=[task_id],
+                    queue_id=queue_id,
+                    reason=f"Task was removed from the queue {queue_id}",
+                )
 
             queue.reload()
             self.metrics.log_queue_metrics_to_es(company_id=company_id, queues=[queue])
@@ -461,7 +525,7 @@ class QueueBLL(object):
                 [
                     {
                         "$match": {
-                            "company": {"$in": [None, "", company]},
+                            "company": {"$in": ["", company]},
                             "_id": queue_id,
                         }
                     },

apiserver/bll/serving/__init__.py

@@ -0,0 +1,376 @@
+from datetime import datetime, timedelta, timezone
+from enum import Enum, auto
+from operator import attrgetter
+from time import time
+from typing import Optional, Sequence, Union
+
+import attr
+from boltons.iterutils import chunked_iter, bucketize
+from pyhocon import ConfigTree
+
+from apiserver.apimodels.serving import (
+    ServingContainerEntry,
+    RegisterRequest,
+    StatusReportRequest,
+)
+from apiserver.apimodels.workers import MachineStats
+from apiserver.apierrors import errors
+from apiserver.config_repo import config
+from apiserver.redis_manager import redman
+from .stats import ServingStats
+
+
+log = config.logger(__file__)
+
+
+class ServingBLL:
+    def __init__(self, redis=None):
+        self.conf = config.get("services.serving", ConfigTree())
+        self.redis = redis or redman.connection("workers")
+
+    @staticmethod
+    def _get_url_key(company: str, url: str):
+        return f"serving_url_{company}_{url}"
+
+    @staticmethod
+    def _get_container_key(company: str, container_id: str) -> str:
+        """Build redis key from company and container_id"""
+        return f"serving_container_{company}_{container_id}"
+
+    def _save_serving_container_entry(self, entry: ServingContainerEntry):
+        self.redis.setex(
+            entry.key, timedelta(seconds=entry.register_timeout), entry.to_json()
+        )
+
+        url_key = self._get_url_key(entry.company_id, entry.endpoint_url)
+        expiration = int(time()) + entry.register_timeout
+        container_item = {entry.key: expiration}
+        self.redis.zadd(url_key, container_item)
+        # make sure that url set will not get stuck in redis
+        # indefinitely in case no more containers report to it
+        self.redis.expire(url_key, max(3600, entry.register_timeout))
+
+    def _get_serving_container_entry(
+        self, company_id: str, container_id: str
+    ) -> Optional[ServingContainerEntry]:
+        """
+        Get a container entry for the provided container ID.
+        """
+        key = self._get_container_key(company_id, container_id)
+        data = self.redis.get(key)
+        if not data:
+            return
+
+        try:
+            entry = ServingContainerEntry.from_json(data)
+            return entry
+        except Exception as e:
+            msg = "Failed parsing container entry"
+            log.exception(f"{msg}: {str(e)}")
+
+    def register_serving_container(
+        self,
+        company_id: str,
+        request: RegisterRequest,
+        ip: str = "",
+    ) -> ServingContainerEntry:
+        """
+        Register a serving container
+        """
+        now = datetime.now(timezone.utc)
+        key = self._get_container_key(company_id, request.container_id)
+        entry = ServingContainerEntry(
+            **request.to_struct(),
+            key=key,
+            company_id=company_id,
+            ip=ip,
+            register_time=now,
+            register_timeout=request.timeout,
+            last_activity_time=now,
+        )
+        self._save_serving_container_entry(entry)
+        return entry
+
+    def unregister_serving_container(
+        self,
+        company_id: str,
+        container_id: str,
+    ) -> None:
+        """
+        Unregister a serving container
+        """
+        entry = self._get_serving_container_entry(company_id, container_id)
+        if entry:
+            url_key = self._get_url_key(entry.company_id, entry.endpoint_url)
+            self.redis.zrem(url_key, entry.key)
+
+        key = self._get_container_key(company_id, container_id)
+        res = self.redis.delete(key)
+        if res:
+            return
+
+        if not self.conf.get("container_auto_unregister", True):
+            raise errors.bad_request.ContainerNotRegistered(container=container_id)
+
+    def container_status_report(
+        self,
+        company_id: str,
+        report: StatusReportRequest,
+        ip: str = "",
+    ) -> None:
+        """
+        Serving container status report
+        """
+        container_id = report.container_id
+        now = datetime.now(timezone.utc)
+        entry = self._get_serving_container_entry(company_id, container_id)
+        if entry:
+            ip = ip or entry.ip
+            register_time = entry.register_time
+            register_timeout = entry.register_timeout
+        else:
+            if not self.conf.get("container_auto_register", True):
+                raise errors.bad_request.ContainerNotRegistered(container=container_id)
+            ip = ip
+            register_time = now
+            register_timeout = int(
+                self.conf.get("default_container_timeout_sec", 10 * 60)
+            )
+
+        key = self._get_container_key(company_id, container_id)
+        entry = ServingContainerEntry(
+            **report.to_struct(),
+            key=key,
+            company_id=company_id,
+            ip=ip,
+            register_time=register_time,
+            register_timeout=register_timeout,
+            last_activity_time=now,
+        )
+        self._save_serving_container_entry(entry)
+        ServingStats.log_stats_to_es(entry)
+
+    def _get_all(
+        self,
+        company_id: str,
+    ) -> Sequence[ServingContainerEntry]:
+        keys = list(self.redis.scan_iter(self._get_container_key(company_id, "*")))
+        entries = []
+        for keys in chunked_iter(keys, 1000):
+            data = self.redis.mget(keys)
+            if not data:
+                continue
+            for d in data:
+                try:
+                    entries.append(ServingContainerEntry.from_json(d))
+                except Exception as ex:
+                    log.error(f"Failed parsing container entry {str(ex)}")
+
+        return entries
+
+    @attr.s(auto_attribs=True)
+    class Counter:
+        class AggType(Enum):
+            avg = auto()
+            max = auto()
+            total = auto()
+            count = auto()
+
+        name: str
+        field: str
+        agg_type: AggType
+        float_precision: int = None
+
+        _max: Union[int, float, datetime] = attr.field(init=False, default=None)
+        _total: Union[int, float] = attr.field(init=False, default=0)
+        _count: int = attr.field(init=False, default=0)
+
+        def add(self, entry: ServingContainerEntry):
+            value = getattr(entry, self.field, None)
+            if value is None:
+                return
+
+            self._count += 1
+            if self.agg_type == self.AggType.max:
+                self._max = value if self._max is None else max(self._max, value)
+            else:
+                self._total += value
+
+        def __call__(self):
+            if self.agg_type == self.AggType.count:
+                return self._count
+
+            if self.agg_type == self.AggType.max:
+                return self._max
+
+            if self.agg_type == self.AggType.total:
+                return self._total
+
+            if not self._count:
+                return None
+            avg = self._total / self._count
+            return (
+                round(avg, self.float_precision) if self.float_precision else round(avg)
+            )
+
+    def _get_summary(self, entries: Sequence[ServingContainerEntry]) -> dict:
+        counters = [
+            self.Counter(
+                name="uptime_sec",
+                field="uptime_sec",
+                agg_type=self.Counter.AggType.max,
+            ),
+            self.Counter(
+                name="requests",
+                field="requests_num",
+                agg_type=self.Counter.AggType.total,
+            ),
+            self.Counter(
+                name="requests_min",
+                field="requests_min",
+                agg_type=self.Counter.AggType.avg,
+                float_precision=2,
+            ),
+            self.Counter(
+                name="latency_ms",
+                field="latency_ms",
+                agg_type=self.Counter.AggType.avg,
+            ),
+            self.Counter(
+                name="last_update",
+                field="last_activity_time",
+                agg_type=self.Counter.AggType.max,
+            ),
+        ]
+        for entry in entries:
+            for counter in counters:
+                counter.add(entry)
+
+        first_entry = entries[0]
+        ret = {
+            "endpoint": first_entry.endpoint_name,
+            "model": first_entry.model_name,
+            "url": first_entry.endpoint_url,
+            "instances": len(entries),
+            **{counter.name: counter() for counter in counters},
+        }
+        ret["last_update"] = ret.get("last_update")
+        return ret
+
+    def get_endpoints(self, company_id: str):
+        """
+        Group instances by urls and return a summary for each url
+        Do not return data for "loading" instances that have no url
+        """
+        entries = self._get_all(company_id)
+        by_url = bucketize(entries, key=attrgetter("endpoint_url"))
+        by_url.pop(None, None)
+        return [self._get_summary(url_entries) for url_entries in by_url.values()]
+
+    def _get_endpoint_entries(
+        self, company_id, endpoint_url: Union[str, None]
+    ) -> Sequence[ServingContainerEntry]:
+        url_key = self._get_url_key(company_id, endpoint_url)
+        timestamp = int(time())
+        self.redis.zremrangebyscore(url_key, min=0, max=timestamp)
+        container_keys = {key.decode() for key in self.redis.zrange(url_key, 0, -1)}
+        if not container_keys:
+            return []
+
+        entries = []
+        found_keys = set()
+        data = self.redis.mget(container_keys) or []
+        for d in data:
+            try:
+                entry = ServingContainerEntry.from_json(d)
+                if entry.endpoint_url == endpoint_url:
+                    entries.append(entry)
+                    found_keys.add(entry.key)
+            except Exception as ex:
+                log.error(f"Failed parsing container entry {str(ex)}")
+
+        missing_keys = container_keys - found_keys
+        if missing_keys:
+            self.redis.zrem(url_key, *missing_keys)
+
+        return entries
+
+    def get_loading_instances(self, company_id: str):
+        entries = self._get_endpoint_entries(company_id, None)
+        return [
+            {
+                "id": entry.container_id,
+                "endpoint": entry.endpoint_name,
+                "url": entry.endpoint_url,
+                "model": entry.model_name,
+                "model_source": entry.model_source,
+                "model_version": entry.model_version,
+                "preprocess_artifact": entry.preprocess_artifact,
+                "input_type": entry.input_type,
+                "input_size": entry.input_size,
+                "uptime_sec": entry.uptime_sec,
+                "age_sec": int((datetime.now(timezone.utc) - entry.register_time).total_seconds()),
+                "last_update": entry.last_activity_time,
+            }
+            for entry in entries
+        ]
+
+    def get_endpoint_details(self, company_id, endpoint_url: str) -> dict:
+        entries = self._get_endpoint_entries(company_id, endpoint_url)
+        if not entries:
+            raise errors.bad_request.NoContainersForUrl(url=endpoint_url)
+
+        instances = []
+        entry: ServingContainerEntry
+        for entry in entries:
+            instances.append(
+                {
+                    "endpoint": entry.endpoint_name,
+                    "model": entry.model_name,
+                    "url": entry.endpoint_url,
+                }
+            )
+
+        def get_machine_stats_data(machine_stats: MachineStats) -> dict:
+            ret = {"cpu_count": 0, "gpu_count": 0}
+            if not machine_stats:
+                return ret
+
+            for value, field in (
+                (machine_stats.cpu_usage, "cpu_count"),
+                (machine_stats.gpu_usage, "gpu_count"),
+            ):
+                if value is None:
+                    continue
+                ret[field] = len(value) if isinstance(value, (list, tuple)) else 1
+
+            return ret
+
+        first_entry = entries[0]
+        return {
+            "endpoint": first_entry.endpoint_name,
+            "model": first_entry.model_name,
+            "url": first_entry.endpoint_url,
+            "preprocess_artifact": first_entry.preprocess_artifact,
+            "input_type": first_entry.input_type,
+            "input_size": first_entry.input_size,
+            "model_source": first_entry.model_source,
+            "model_version": first_entry.model_version,
+            "uptime_sec": max(e.uptime_sec for e in entries),
+            "last_update": max(e.last_activity_time for e in entries),
+            "instances": [
+                {
+                    "id": entry.container_id,
+                    "uptime_sec": entry.uptime_sec,
+                    "requests": entry.requests_num,
+                    "requests_min": entry.requests_min,
+                    "latency_ms": entry.latency_ms,
+                    "last_update": entry.last_activity_time,
+                    "reference": [ref.to_struct() for ref in entry.reference]
+                    if isinstance(entry.reference, list)
+                    else entry.reference,
+                    **get_machine_stats_data(entry.machine_stats),
+                }
+                for entry in entries
+            ],
+        }

apiserver/bll/serving/stats.py

@@ -0,0 +1,335 @@
+from collections import defaultdict
+from datetime import datetime, timezone
+from enum import Enum
+
+from typing import Tuple, Optional, Sequence
+
+from elasticsearch import Elasticsearch
+
+from apiserver.apimodels.serving import (
+    ServingContainerEntry,
+    GetEndpointMetricsHistoryRequest,
+    MetricType,
+)
+from apiserver.apierrors import errors
+from apiserver.utilities.dicts import nested_get
+from apiserver.bll.query import Builder as QueryBuilder
+from apiserver.config_repo import config
+from apiserver.es_factory import es_factory
+
+
+class _AggregationType(Enum):
+    avg = "avg"
+    sum = "sum"
+
+
+class ServingStats:
+    min_chart_interval = config.get("services.serving.min_chart_interval_sec", 40)
+    es: Elasticsearch = es_factory.connect("workers")
+
+    @classmethod
+    def _serving_stats_prefix(cls, company_id: str) -> str:
+        """Returns the es index prefix for the company"""
+        return f"serving_stats_{company_id.lower()}_"
+
+    @staticmethod
+    def _get_es_index_suffix():
+        """Get the index name suffix for storing current month data"""
+        return datetime.now(timezone.utc).strftime("%Y-%m")
+
+    @staticmethod
+    def _get_average_value(value) -> Tuple[Optional[float], Optional[int]]:
+        if value is None:
+            return None, None
+
+        if isinstance(value, (list, tuple)):
+            count = len(value)
+            if not count:
+                return None, None
+            return sum(value) / count, count
+
+        return value, 1
+
+    @classmethod
+    def log_stats_to_es(
+        cls,
+        entry: ServingContainerEntry,
+    ) -> int:
+        """
+        Actually writing the worker statistics to Elastic
+        :return: The amount of logged documents
+        """
+        company_id = entry.company_id
+        es_index = (
+            f"{cls._serving_stats_prefix(company_id)}" f"{cls._get_es_index_suffix()}"
+        )
+
+        entry_data = entry.to_struct()
+        doc = {
+            "timestamp": es_factory.get_timestamp_millis(),
+            **{
+                field: entry_data.get(field)
+                for field in (
+                    "container_id",
+                    "company_id",
+                    "endpoint_url",
+                    "requests_num",
+                    "requests_min",
+                    "uptime_sec",
+                    "latency_ms",
+                )
+            },
+        }
+
+        stats = entry_data.get("machine_stats")
+        if stats:
+            for category in ("cpu", "gpu"):
+                usage, num = cls._get_average_value(stats.get(f"{category}_usage"))
+                doc.update({f"{category}_usage": usage, f"{category}_num": num})
+
+            for category in ("memory", "gpu_memory"):
+                free, _ = cls._get_average_value(stats.get(f"{category}_free"))
+                used, _ = cls._get_average_value(stats.get(f"{category}_used"))
+                doc.update(
+                    {
+                        f"{category}_free": free,
+                        f"{category}_used": used,
+                        f"{category}_total": round((free or 0) + (used or 0), 3),
+                    }
+                )
+
+            doc.update(
+                {
+                    field: stats.get(field)
+                    for field in ("disk_free_home", "network_rx", "network_tx")
+                }
+            )
+
+        cls.es.index(index=es_index, document=doc)
+
+        return 1
+
+    @staticmethod
+    def round_series(values: Sequence, koeff) -> list:
+        return [round(v * koeff, 2) if v else 0 for v in values]
+
+    _mb_to_gb = 1 / 1024
+    agg_fields = {
+        MetricType.requests: (
+            "requests_num",
+            "Number of Requests",
+            _AggregationType.sum,
+            None,
+        ),
+        MetricType.requests_min: (
+            "requests_min",
+            "Requests per Minute",
+            _AggregationType.sum,
+            None,
+        ),
+        MetricType.latency_ms: (
+            "latency_ms",
+            "Average Latency (ms)",
+            _AggregationType.avg,
+            None,
+        ),
+        MetricType.cpu_count: ("cpu_num", "CPU Count", _AggregationType.sum, None),
+        MetricType.gpu_count: ("gpu_num", "GPU Count", _AggregationType.sum, None),
+        MetricType.cpu_util: (
+            "cpu_usage",
+            "Average CPU Load (%)",
+            _AggregationType.avg,
+            None,
+        ),
+        MetricType.gpu_util: (
+            "gpu_usage",
+            "Average GPU Utilization (%)",
+            _AggregationType.avg,
+            None,
+        ),
+        MetricType.ram_total: (
+            "memory_total",
+            "RAM Total (GB)",
+            _AggregationType.sum,
+            _mb_to_gb,
+        ),
+        MetricType.ram_used: (
+            "memory_used",
+            "RAM Used (GB)",
+            _AggregationType.sum,
+            _mb_to_gb,
+        ),
+        MetricType.ram_free: (
+            "memory_free",
+            "RAM Free (GB)",
+            _AggregationType.sum,
+            _mb_to_gb,
+        ),
+        MetricType.gpu_ram_total: (
+            "gpu_memory_total",
+            "GPU RAM Total (GB)",
+            _AggregationType.sum,
+            _mb_to_gb,
+        ),
+        MetricType.gpu_ram_used: (
+            "gpu_memory_used",
+            "GPU RAM Used (GB)",
+            _AggregationType.sum,
+            _mb_to_gb,
+        ),
+        MetricType.gpu_ram_free: (
+            "gpu_memory_free",
+            "GPU RAM Free (GB)",
+            _AggregationType.sum,
+            _mb_to_gb,
+        ),
+        MetricType.network_rx: (
+            "network_rx",
+            "Network Throughput RX (MBps)",
+            _AggregationType.sum,
+            None,
+        ),
+        MetricType.network_tx: (
+            "network_tx",
+            "Network Throughput TX (MBps)",
+            _AggregationType.sum,
+            None,
+        ),
+    }
+
+    @classmethod
+    def get_endpoint_metrics(
+        cls,
+        company_id: str,
+        metrics_request: GetEndpointMetricsHistoryRequest,
+    ) -> dict:
+        from_date = metrics_request.from_date
+        to_date = metrics_request.to_date
+        if from_date >= to_date:
+            raise errors.bad_request.FieldsValueError(
+                "from_date must be less than to_date"
+            )
+
+        metric_type = metrics_request.metric_type
+        agg_data = cls.agg_fields.get(metric_type)
+        if not agg_data:
+            raise NotImplemented(f"Charts for {metric_type} not implemented")
+
+        agg_field, title, agg_type, multiplier = agg_data
+        if agg_type == _AggregationType.sum:
+            instance_sum_type = "sum_bucket"
+        else:
+            instance_sum_type = "avg_bucket"
+
+        interval = max(metrics_request.interval, cls.min_chart_interval)
+        endpoint_url = metrics_request.endpoint_url
+        hist_ret = {
+            "computed_interval": interval,
+            "total": {
+                "title": title,
+                "dates": [],
+                "values": [],
+            },
+            "instances": {},
+        }
+        must_conditions = [
+            QueryBuilder.term("company_id", company_id),
+            QueryBuilder.term("endpoint_url", endpoint_url),
+            QueryBuilder.dates_range(from_date, to_date),
+        ]
+        query = {"bool": {"must": must_conditions}}
+        es_index = f"{cls._serving_stats_prefix(company_id)}*"
+        res = cls.es.search(
+            index=es_index,
+            size=0,
+            query=query,
+            aggs={"instances": {"terms": {"field": "container_id"}}},
+        )
+        instance_buckets = nested_get(res, ("aggregations", "instances", "buckets"))
+        if not instance_buckets:
+            return hist_ret
+
+        instance_keys = {ib["key"] for ib in instance_buckets}
+        must_conditions.append(QueryBuilder.terms("container_id", instance_keys))
+        query = {"bool": {"must": must_conditions}}
+        sample_func = "avg" if metric_type != MetricType.requests else "max"
+        aggs = {
+            "instances": {
+                "terms": {
+                    "field": "container_id",
+                    "size": max(len(instance_keys), 10),
+                },
+                "aggs": {
+                    "sample": {sample_func: {"field": agg_field}},
+                },
+            },
+            "total_instances": {
+                instance_sum_type: {
+                    "gap_policy": "insert_zeros",
+                    "buckets_path": "instances>sample",
+                }
+            },
+        }
+        aggs = {
+            "dates": {
+                "date_histogram": {
+                    "field": "timestamp",
+                    "fixed_interval": f"{interval}s",
+                    "extended_bounds": {
+                        "min": int(from_date) * 1000,
+                        "max": int(to_date) * 1000,
+                    },
+                },
+                "aggs": aggs,
+            }
+        }
+
+        filter_path = None
+        if not metrics_request.instance_charts:
+            filter_path = "aggregations.dates.buckets.total_instances"
+
+        data = cls.es.search(
+            index=es_index,
+            size=0,
+            query=query,
+            aggs=aggs,
+            filter_path=filter_path,
+        )
+        agg_res = data.get("aggregations")
+        if not agg_res:
+            return hist_ret
+
+        dates_ = []
+        total = []
+        instances = defaultdict(list)
+        # remove last interval if it's incomplete. Allow 10% tolerance
+        last_valid_timestamp = (to_date - 0.9 * interval) * 1000
+        for point in agg_res["dates"]["buckets"]:
+            date_ = point["key"]
+            if date_ > last_valid_timestamp:
+                break
+            dates_.append(date_)
+            total.append(nested_get(point, ("total_instances", "value"), 0))
+            if metrics_request.instance_charts:
+                found_keys = set()
+                for instance in nested_get(point, ("instances", "buckets"), []):
+                    instances[instance["key"]].append(
+                        nested_get(instance, ("sample", "value"), 0)
+                    )
+                    found_keys.add(instance["key"])
+                for missing_key in instance_keys - found_keys:
+                    instances[missing_key].append(0)
+
+        koeff = multiplier if multiplier else 1.0
+        hist_ret["total"]["dates"] = dates_
+        hist_ret["total"]["values"] = cls.round_series(total, koeff)
+        hist_ret["instances"] = {
+            key: {
+                "title": key,
+                "dates": dates_,
+                "values": cls.round_series(values, koeff),
+            }
+            for key, values in sorted(instances.items(), key=lambda p: p[0])
+        }
+
+        return hist_ret

apiserver/bll/statistics/stats_reporter.py

@@ -18,7 +18,7 @@ from apiserver.config.info import get_deployment_type
 from apiserver.database.model import Company, User
 from apiserver.database.model.queue import Queue
 from apiserver.database.model.task.task import Task
-from apiserver.tools import safe_get
+from apiserver.utilities.dicts import nested_get
 from apiserver.utilities.json import dumps
 from apiserver.version import __version__ as current_version
 from .resource_monitor import ResourceMonitor, stat_threads
@@ -162,7 +162,7 @@ class StatisticsReporter:
         def _get_cardinality_fields(categories: Sequence[dict]) -> dict:
             names = {"cpu": "num_cores"}
             return {
-                names[c["key"]]: safe_get(c, "count/value")
+                names[c["key"]]: nested_get(c, ("count", "value"))
                 for c in categories
                 if c["key"] in names
             }
@@ -175,21 +175,21 @@ class StatisticsReporter:
             }
             return {
                 names[m["key"]]: {
-                    "min": safe_get(m, "min/value"),
-                    "max": safe_get(m, "max/value"),
-                    "avg": safe_get(m, "avg/value"),
+                    "min": nested_get(m, ("min", "value")),
+                    "max": nested_get(m, ("max", "value")),
+                    "avg": nested_get(m, ("avg", "value")),
                 }
                 for m in metrics
                 if m["key"] in names
             }
 
-        buckets = safe_get(res, "aggregations/workers/buckets", default=[])
+        buckets = nested_get(res, ("aggregations", "workers", "buckets"), default=[])
         return {
             b["key"]: {
                 key: {
                     "interval_sec": agent_resource_threshold_sec,
-                    **_get_cardinality_fields(safe_get(b, "categories/buckets", [])),
-                    **_get_metric_fields(safe_get(b, "metrics/buckets", [])),
+                    **_get_cardinality_fields(nested_get(b, ("categories", "buckets"), [])),
+                    **_get_metric_fields(nested_get(b, ("metrics", "buckets"), [])),
                 }
             }
             for b in buckets
@@ -227,7 +227,7 @@ class StatisticsReporter:
             },
         }
         res = cls._run_worker_stats_query(company_id, es_req)
-        buckets = safe_get(res, "aggregations/workers/buckets", default=[])
+        buckets = nested_get(res, ("aggregations", "workers", "buckets"), default=[])
         return {
             b["key"]: {"last_activity_time": b["last_activity_time"]["value"]}
             for b in buckets
@@ -254,6 +254,14 @@ class StatisticsReporter:
                     **({"last_worker": {"$in": workers}} if workers else {}),
                 }
             },
+            {
+                "$project": {
+                    "last_worker": 1,
+                    "last_update": 1,
+                    "started": 1,
+                    "last_iteration": 1,
+                }
+            },
             {
                 "$group": {
                     "_id": "$last_worker" if workers else None,

apiserver/bll/storage/__init__.py

@@ -1,14 +1,32 @@
+import json
+import os
+import tempfile
 from copy import copy
+from datetime import datetime
+from typing import Optional, Sequence
 
+import attr
 from boltons.cacheutils import cachedproperty
 from clearml.backend_config.bucket_config import (
     S3BucketConfigurations,
     AzureContainerConfigurations,
     GSBucketConfigurations,
+    AzureContainerConfig,
+    GSBucketConfig,
+    S3BucketConfig,
 )
 
+from apiserver.apierrors import errors
+from apiserver.apimodels.storage import SetSettingsRequest
 from apiserver.config_repo import config
-
+from apiserver.database.model.storage_settings import (
+    StorageSettings,
+    GoogleBucketSettings,
+    AWSSettings,
+    AzureStorageSettings,
+    GoogleStorageSettings,
+)
+from apiserver.database.utils import id as db_id
 
 log = config.logger(__file__)
 
@@ -32,17 +50,224 @@ class StorageBLL:
     def get_azure_settings_for_company(
         self,
         company_id: str,
+        db_settings: StorageSettings = None,
+        query_db: bool = True,
     ) -> AzureContainerConfigurations:
-        return copy(self._default_azure_configs)
+        if not db_settings and query_db:
+            db_settings = (
+                StorageSettings.objects(company=company_id).only("azure").first()
+            )
+
+        if not db_settings or not db_settings.azure:
+            return copy(self._default_azure_configs)
+
+        azure = db_settings.azure
+        return AzureContainerConfigurations(
+            container_configs=[
+                AzureContainerConfig(**entry.to_proper_dict())
+                for entry in (azure.containers or [])
+            ]
+        )
 
     def get_gs_settings_for_company(
         self,
         company_id: str,
+        db_settings: StorageSettings = None,
+        query_db: bool = True,
+        json_string: bool = False,
     ) -> GSBucketConfigurations:
-        return copy(self._default_gs_configs)
+        if not db_settings and query_db:
+            db_settings = (
+                StorageSettings.objects(company=company_id).only("google").first()
+            )
+
+        if not db_settings or not db_settings.google:
+            if not json_string:
+                return copy(self._default_gs_configs)
+
+            if self._default_gs_configs._buckets:
+                buckets = [
+                    attr.evolve(
+                        b,
+                        credentials_json=self._assure_json_string(b.credentials_json),
+                    )
+                    for b in self._default_gs_configs._buckets
+                ]
+            else:
+                buckets = self._default_gs_configs._buckets
+
+            return GSBucketConfigurations(
+                buckets=buckets,
+                default_project=self._default_gs_configs._default_project,
+                default_credentials=self._assure_json_string(
+                    self._default_gs_configs._default_credentials
+                ),
+            )
+
+        def get_bucket_config(bc: GoogleBucketSettings) -> GSBucketConfig:
+            data = bc.to_proper_dict()
+            if not json_string and bc.credentials_json:
+                data["credentials_json"] = self._assure_json_file(bc.credentials_json)
+            return GSBucketConfig(**data)
+
+        google = db_settings.google
+        buckets_configs = [get_bucket_config(b) for b in (google.buckets or [])]
+        return GSBucketConfigurations(
+            buckets=buckets_configs,
+            default_project=google.project,
+            default_credentials=google.credentials_json
+            if json_string
+            else self._assure_json_file(google.credentials_json),
+        )
 
     def get_aws_settings_for_company(
         self,
         company_id: str,
+        db_settings: StorageSettings = None,
+        query_db: bool = True,
     ) -> S3BucketConfigurations:
-        return copy(self._default_aws_configs)
+        if not db_settings and query_db:
+            db_settings = (
+                StorageSettings.objects(company=company_id).only("aws").first()
+            )
+        if not db_settings or not db_settings.aws:
+            return copy(self._default_aws_configs)
+
+        aws = db_settings.aws
+        buckets_configs = S3BucketConfig.from_list(
+            [b.to_proper_dict() for b in (aws.buckets or [])]
+        )
+        return S3BucketConfigurations(
+            buckets=buckets_configs,
+            default_key=aws.key,
+            default_secret=aws.secret,
+            default_region=aws.region,
+            default_use_credentials_chain=aws.use_credentials_chain,
+            default_token=aws.token,
+            default_extra_args={},
+        )
+
+    def _assure_json_file(self, name_or_content: str) -> str:
+        if not name_or_content:
+            return name_or_content
+
+        if name_or_content.endswith(".json") or os.path.exists(name_or_content):
+            return name_or_content
+
+        try:
+            json.loads(name_or_content)
+        except Exception:
+            return name_or_content
+
+        with tempfile.NamedTemporaryFile(
+            mode="wt", delete=False, suffix=".json"
+        ) as tmp:
+            tmp.write(name_or_content)
+
+        return tmp.name
+
+    def _assure_json_string(self, name_or_content: str) -> Optional[str]:
+        if not name_or_content:
+            return name_or_content
+
+        try:
+            json.loads(name_or_content)
+            return name_or_content
+        except Exception:
+            pass
+
+        try:
+            with open(name_or_content) as fp:
+                return fp.read()
+        except Exception:
+            return None
+
+    def get_company_settings(self, company_id: str) -> dict:
+        db_settings = StorageSettings.objects(company=company_id).first()
+        aws = self.get_aws_settings_for_company(company_id, db_settings, query_db=False)
+        aws_dict = {
+            "key": aws._default_key,
+            "secret": aws._default_secret,
+            "token": aws._default_token,
+            "region": aws._default_region,
+            "use_credentials_chain": aws._default_use_credentials_chain,
+            "buckets": [attr.asdict(b) for b in aws._buckets],
+        }
+
+        gs = self.get_gs_settings_for_company(
+            company_id, db_settings, query_db=False, json_string=True
+        )
+        gs_dict = {
+            "project": gs._default_project,
+            "credentials_json": gs._default_credentials,
+            "buckets": [attr.asdict(b) for b in gs._buckets],
+        }
+
+        azure = self.get_azure_settings_for_company(company_id, db_settings)
+        azure_dict = {
+            "containers": [attr.asdict(ac) for ac in azure._container_configs],
+        }
+
+        return {
+            "aws": aws_dict,
+            "google": gs_dict,
+            "azure": azure_dict,
+            "last_update": db_settings.last_update if db_settings else None,
+        }
+
+    def set_company_settings(
+        self, company_id: str, settings: SetSettingsRequest
+    ) -> int:
+        update_dict = {}
+        if settings.aws:
+            update_dict["aws"] = {
+                **{
+                    k: v
+                    for k, v in settings.aws.to_struct().items()
+                    if k in AWSSettings.get_fields()
+                }
+            }
+
+        if settings.azure:
+            update_dict["azure"] = {
+                **{
+                    k: v
+                    for k, v in settings.azure.to_struct().items()
+                    if k in AzureStorageSettings.get_fields()
+                }
+            }
+
+        if settings.google:
+            update_dict["google"] = {
+                **{
+                    k: v
+                    for k, v in settings.google.to_struct().items()
+                    if k in GoogleStorageSettings.get_fields()
+                }
+            }
+            cred_json = update_dict["google"].get("credentials_json")
+            if cred_json:
+                try:
+                    json.loads(cred_json)
+                except Exception as ex:
+                    raise errors.bad_request.ValidationError(
+                        f"Invalid json credentials: {str(ex)}"
+                    )
+
+        if not update_dict:
+            raise errors.bad_request.ValidationError("No settings were provided")
+
+        settings = StorageSettings.objects(company=company_id).only("id").first()
+        settings_id = settings.id if settings else db_id()
+        return StorageSettings.objects(id=settings_id).update(
+            upsert=True,
+            id=settings_id,
+            company=company_id,
+            last_update=datetime.utcnow(),
+            **update_dict,
+        )
+
+    def reset_company_settings(self, company_id: str, keys: Sequence[str]) -> int:
+        return StorageSettings.objects(company=company_id).update(
+            last_update=datetime.utcnow(), **{f"unset__{k}": 1 for k in keys}
+        )

apiserver/bll/task/__init__.py

@@ -1,6 +1,5 @@
 from .task_bll import TaskBLL
 from .utils import (
     ChangeStatusRequest,
-    update_project_time,
     validate_status_change,
 )

apiserver/bll/task/artifacts.py

@@ -5,6 +5,7 @@ from apiserver.apimodels.tasks import Artifact as ApiArtifact, ArtifactId
 from apiserver.bll.task.utils import get_task_for_update, update_task
 from apiserver.database.model.task.task import DEFAULT_ARTIFACT_MODE, Artifact
 from apiserver.database.utils import hash_field_name
+from apiserver.service_repo.auth import Identity
 from apiserver.utilities.dicts import nested_get, nested_set
 from apiserver.utilities.parameter_key_escaper import mongoengine_safe
 
@@ -48,12 +49,14 @@ class Artifacts:
     def add_or_update_artifacts(
         cls,
         company_id: str,
-        user_id: str,
+        identity: Identity,
         task_id: str,
         artifacts: Sequence[ApiArtifact],
         force: bool,
     ) -> int:
-        task = get_task_for_update(company_id=company_id, task_id=task_id, force=force,)
+        task = get_task_for_update(
+            company_id=company_id, task_id=task_id, force=force, identity=identity
+        )
 
         artifacts = {
             get_artifact_id(a): Artifact(**a)
@@ -64,18 +67,20 @@ class Artifacts:
             f"set__execution__artifacts__{mongoengine_safe(name)}": value
             for name, value in artifacts.items()
         }
-        return update_task(task, user_id=user_id, update_cmds=update_cmds)
+        return update_task(task, user_id=identity.user, update_cmds=update_cmds)
 
     @classmethod
     def delete_artifacts(
         cls,
         company_id: str,
-        user_id: str,
+        identity: Identity,
         task_id: str,
         artifact_ids: Sequence[ArtifactId],
         force: bool,
     ) -> int:
-        task = get_task_for_update(company_id=company_id, task_id=task_id, force=force,)
+        task = get_task_for_update(
+            company_id=company_id, task_id=task_id, force=force, identity=identity
+        )
 
         artifact_ids = [
             get_artifact_id(a)
@@ -85,4 +90,4 @@ class Artifacts:
             f"unset__execution__artifacts__{id_}": 1 for id_ in set(artifact_ids)
         }
 
-        return update_task(task, user_id=user_id, update_cmds=delete_cmds)
+        return update_task(task, user_id=identity.user, update_cmds=delete_cmds)

apiserver/bll/task/hyperparams.py

@@ -15,6 +15,7 @@ from apiserver.bll.task import TaskBLL
 from apiserver.bll.task.utils import get_task_for_update, update_task
 from apiserver.config_repo import config
 from apiserver.database.model.task.task import ParamsItem, Task, ConfigurationItem
+from apiserver.service_repo.auth import Identity
 from apiserver.utilities.parameter_key_escaper import (
     ParameterKeyEscaper,
     mongoengine_safe,
@@ -31,7 +32,10 @@ class HyperParams:
     def get_params(cls, company_id: str, task_ids: Sequence[str]) -> Dict[str, dict]:
         only = ("id", "hyperparams")
         tasks = task_bll.assert_exists(
-            company_id=company_id, task_ids=task_ids, only=only, allow_public=True,
+            company_id=company_id,
+            task_ids=task_ids,
+            only=only,
+            allow_public=True,
         )
 
         return {
@@ -63,7 +67,7 @@ class HyperParams:
     def delete_params(
         cls,
         company_id: str,
-        user_id: str,
+        identity: Identity,
         task_id: str,
         hyperparams: Sequence[HyperParamKey],
         force: bool,
@@ -74,6 +78,7 @@ class HyperParams:
             task_id=task_id,
             allow_all_statuses=properties_only,
             force=force,
+            identity=identity,
         )
 
         with_param, without_param = iterutils.partition(
@@ -96,7 +101,7 @@ class HyperParams:
 
         return update_task(
             task,
-            user_id=user_id,
+            user_id=identity.user,
             update_cmds=delete_cmds,
             set_last_update=not properties_only,
         )
@@ -105,7 +110,7 @@ class HyperParams:
     def edit_params(
         cls,
         company_id: str,
-        user_id: str,
+        identity: Identity,
         task_id: str,
         hyperparams: Sequence[HyperParamItem],
         replace_hyperparams: str,
@@ -117,6 +122,7 @@ class HyperParams:
             task_id=task_id,
             allow_all_statuses=properties_only,
             force=force,
+            identity=identity,
         )
 
         update_cmds = dict()
@@ -135,7 +141,7 @@ class HyperParams:
 
         return update_task(
             task,
-            user_id=user_id,
+            user_id=identity.user,
             update_cmds=update_cmds,
             set_last_update=not properties_only,
         )
@@ -163,7 +169,10 @@ class HyperParams:
         else:
             only.append("configuration")
         tasks = task_bll.assert_exists(
-            company_id=company_id, task_ids=task_ids, only=only, allow_public=True,
+            company_id=company_id,
+            task_ids=task_ids,
+            only=only,
+            allow_public=True,
         )
 
         return {
@@ -184,7 +193,7 @@ class HyperParams:
         pipeline = [
             {
                 "$match": {
-                    "company": {"$in": [None, "", company_id]},
+                    "company": {"$in": ["", company_id]},
                     "_id": {"$in": task_ids},
                 }
             },
@@ -209,13 +218,15 @@ class HyperParams:
     def edit_configuration(
         cls,
         company_id: str,
-        user_id: str,
+        identity: Identity,
         task_id: str,
         configuration: Sequence[Configuration],
         replace_configuration: bool,
         force: bool,
     ) -> int:
-        task = get_task_for_update(company_id=company_id, task_id=task_id, force=force)
+        task = get_task_for_update(
+            company_id=company_id, task_id=task_id, force=force, identity=identity
+        )
 
         update_cmds = dict()
         configuration = {
@@ -228,22 +239,24 @@ class HyperParams:
             for name, value in configuration.items():
                 update_cmds[f"set__configuration__{mongoengine_safe(name)}"] = value
 
-        return update_task(task, user_id=user_id, update_cmds=update_cmds)
+        return update_task(task, user_id=identity.user, update_cmds=update_cmds)
 
     @classmethod
     def delete_configuration(
         cls,
         company_id: str,
-        user_id: str,
+        identity: Identity,
         task_id: str,
         configuration: Sequence[str],
         force: bool,
     ) -> int:
-        task = get_task_for_update(company_id=company_id, task_id=task_id, force=force)
+        task = get_task_for_update(
+            company_id=company_id, task_id=task_id, force=force, identity=identity
+        )
 
         delete_cmds = {
             f"unset__configuration__{ParameterKeyEscaper.escape(name)}": 1
             for name in set(configuration)
         }
 
-        return update_task(task, user_id=user_id, update_cmds=delete_cmds)
+        return update_task(task, user_id=identity.user, update_cmds=delete_cmds)

apiserver/bll/task/non_responsive_tasks_watchdog.py

@@ -1,7 +1,7 @@
 from datetime import timedelta, datetime
 from time import sleep
 
-from apiserver.bll.task import update_project_time
+from apiserver.bll.util import update_project_time
 from apiserver.config_repo import config
 from apiserver.database.model.task.task import TaskStatus, Task
 from apiserver.utilities.threads_manager import ThreadsManager
@@ -85,6 +85,7 @@ class NonResponsiveTasksWatchdog:
                     status_changed=now,
                     last_update=now,
                     last_change=now,
+                    last_changed_by="__apiserver__",
                 )
                 if updated:
                     project_ids.add(task.project)

apiserver/bll/task/task_bll.py

@@ -12,6 +12,7 @@ from apiserver.apimodels.tasks import TaskInputModel
 from apiserver.bll.queue import QueueBLL
 from apiserver.bll.organization import OrgBLL, Tags
 from apiserver.bll.project import ProjectBLL
+from apiserver.bll.util import update_project_time
 from apiserver.config_repo import config
 from apiserver.database.errors import translate_errors_context
 from apiserver.database.model.model import Model
@@ -31,15 +32,18 @@ from apiserver.database.model.task.task import (
 )
 from apiserver.database.model import EntityVisibility
 from apiserver.database.model.queue import Queue
-from apiserver.database.utils import get_company_or_none_constraint, id as create_id
+from apiserver.database.utils import (
+    get_company_or_none_constraint,
+    id as create_id,
+)
 from apiserver.es_factory import es_factory
 from apiserver.redis_manager import redman
 from apiserver.services.utils import validate_tags, escape_dict_field, escape_dict
+from apiserver.utilities.dicts import nested_set
 from .artifacts import artifacts_prepare_for_save
 from .param_utils import params_prepare_for_save
 from .utils import (
     ChangeStatusRequest,
-    update_project_time,
     deleted_prefix,
     get_last_metric_updates,
 )
@@ -55,30 +59,13 @@ class TaskBLL:
         self.events_es = events_es or es_factory.connect("events")
         self.redis: StrictRedis = redis or redman.connection("apiserver")
 
-    @staticmethod
-    def get_task_with_access(
-        task_id, company_id, only=None, allow_public=False, requires_write_access=False
-    ) -> Task:
-        """
-        Gets a task that has a required write access
-        :except errors.bad_request.InvalidTaskId: if the task is not found
-        :except errors.forbidden.NoWritePermission: if write_access was required and the task cannot be modified
-        """
-        with translate_errors_context():
-            query = dict(id=task_id, company=company_id)
-            if requires_write_access:
-                task = Task.get_for_writing(_only=only, **query)
-            else:
-                task = Task.get(_only=only, **query, include_public=allow_public)
-
-            if not task:
-                raise errors.bad_request.InvalidTaskId(**query)
-
-            return task
-
     @staticmethod
     def get_by_id(
-        company_id, task_id, required_status=None, only_fields=None, allow_public=False,
+        company_id,
+        task_id,
+        required_status=None,
+        only_fields=None,
+        allow_public=False,
     ):
         if only_fields:
             if isinstance(only_fields, string_types):
@@ -177,18 +164,36 @@ class TaskBLL:
         input_models: Optional[Sequence[TaskInputModel]] = None,
         validate_references: bool = False,
         new_project_name: str = None,
+        hyperparams_overrides: Optional[dict] = None,
+        configuration_overrides: Optional[dict] = None,
     ) -> Tuple[Task, dict]:
         validate_tags(tags, system_tags)
-        params_dict = {
-            field: value
-            for field, value in (
-                ("hyperparams", hyperparams),
-                ("configuration", configuration),
-            )
-            if value is not None
-        }
+        task: Task = cls.get_by_id(
+            company_id=company_id, task_id=task_id, allow_public=True
+        )
 
-        task = cls.get_by_id(company_id=company_id, task_id=task_id, allow_public=True)
+        params_dict = {}
+        if hyperparams:
+            params_dict["hyperparams"] = hyperparams
+        elif hyperparams_overrides:
+            updated_hyperparams = {
+                sec: {k: value for k, value in sec_data.items()}
+                for sec, sec_data in (task.hyperparams or {}).items()
+            }
+            for section, section_data in hyperparams_overrides.items():
+                for key, value in section_data.items():
+                    nested_set(updated_hyperparams, (section, key), value)
+            params_dict["hyperparams"] = updated_hyperparams
+
+        if configuration:
+            params_dict["configuration"] = configuration
+        elif configuration_overrides:
+            updated_configuration = {
+                k: value for k, value in (task.configuration or {}).items()
+            }
+            for key, value in configuration_overrides.items():
+                updated_configuration[key] = value
+            params_dict["configuration"] = updated_configuration
 
         now = datetime.utcnow()
         if input_models:
@@ -313,7 +318,7 @@ class TaskBLL:
         org_bll.update_tags(
             company_id,
             Tags.Task,
-            project=new_task.project,
+            projects=[new_task.project],
             tags=updated_tags,
             system_tags=updated_system_tags,
         )
@@ -356,6 +361,7 @@ class TaskBLL:
     def set_last_update(
         task_ids: Collection[str],
         company_id: str,
+        user_id: str,
         last_update: datetime,
         **extra_updates,
     ):
@@ -376,6 +382,7 @@ class TaskBLL:
                 upsert=False,
                 last_update=last_update,
                 last_change=last_update,
+                last_changed_by=user_id,
                 **updates,
             )
         return count
@@ -384,6 +391,7 @@ class TaskBLL:
     def update_statistics(
         task_id: str,
         company_id: str,
+        user_id: str,
         last_update: datetime = None,
         last_iteration: int = None,
         last_iteration_max: int = None,
@@ -440,6 +448,7 @@ class TaskBLL:
         ret = TaskBLL.set_last_update(
             task_ids=[task_id],
             company_id=company_id,
+            user_id=user_id,
             last_update=last_update,
             **extra_updates,
         )
@@ -449,8 +458,13 @@ class TaskBLL:
         return ret
 
     @staticmethod
-    def remove_task_from_all_queues(company_id: str, task_id: str) -> int:
-        return Queue.objects(company=company_id, entries__task=task_id).update(
+    def remove_task_from_all_queues(
+        company_id: str, task_id: str, exclude: str = None
+    ) -> int:
+        more = {}
+        if exclude:
+            more["id__ne"] = exclude
+        return Queue.objects(company=company_id, entries__task=task_id, **more).update(
             pull__entries__task=task_id, last_update=datetime.utcnow()
         )
 
@@ -464,9 +478,10 @@ class TaskBLL:
         status_reason: str,
         remove_from_all_queues=False,
         new_status=None,
+        new_status_for_aborted_task=None,
     ):
         try:
-            cls.dequeue(task, company_id, silent_fail=True)
+            cls.dequeue(task, company_id=company_id, user_id=user_id, silent_fail=True)
         except APIError:
             # dequeue may fail if the queue was deleted
             pass
@@ -477,6 +492,9 @@ class TaskBLL:
         if task.status not in [TaskStatus.queued, TaskStatus.in_progress]:
             return {"updated": 0}
 
+        if new_status_for_aborted_task and task.status == TaskStatus.in_progress:
+            new_status = new_status_for_aborted_task
+
         return ChangeStatusRequest(
             task=task,
             new_status=new_status or task.enqueue_status or TaskStatus.created,
@@ -487,7 +505,7 @@ class TaskBLL:
         ).execute(enqueue_status=None)
 
     @classmethod
-    def dequeue(cls, task: Task, company_id: str, silent_fail=False):
+    def dequeue(cls, task: Task, company_id: str, user_id: str, silent_fail=False):
         """
         Dequeue the task from the queue
         :param task: task to dequeue
@@ -514,6 +532,9 @@ class TaskBLL:
 
         return {
             "removed": queue_bll.remove_task(
-                company_id=company_id, queue_id=task.execution.queue, task_id=task.id
+                company_id=company_id,
+                user_id=user_id,
+                queue_id=task.execution.queue,
+                task_id=task.id,
             )
         }

apiserver/bll/task/task_cleanup.py

@@ -1,10 +1,10 @@
 from datetime import datetime
 from itertools import chain
 from operator import attrgetter
-from typing import Sequence, Set, Tuple
+from typing import Sequence, Set, Tuple, Union
 
 import attr
-from boltons.iterutils import partition, bucketize, first
+from boltons.iterutils import partition, bucketize, first, chunked_iter
 from furl import furl
 from mongoengine import NotUniqueError
 from pymongo.errors import DuplicateKeyError
@@ -26,14 +26,13 @@ from apiserver.database.utils import id as db_id
 
 log = config.logger(__file__)
 event_bll = EventBLL()
-async_events_delete = config.get("services.tasks.async_events_delete", False)
 
 
 @attr.s(auto_attribs=True)
 class TaskUrls:
     model_urls: Sequence[str]
-    event_urls: Sequence[str]
     artifact_urls: Sequence[str]
+    event_urls: Sequence[str] = []  # left here is in order not to break the api
 
     def __add__(self, other: "TaskUrls"):
         if not other:
@@ -41,7 +40,6 @@ class TaskUrls:
 
         return TaskUrls(
             model_urls=list(set(self.model_urls) | set(other.model_urls)),
-            event_urls=list(set(self.event_urls) | set(other.event_urls)),
             artifact_urls=list(set(self.artifact_urls) | set(other.artifact_urls)),
         )
 
@@ -55,8 +53,23 @@ class CleanupResult:
     updated_children: int
     updated_models: int
     deleted_models: int
+    deleted_model_ids: Set[str]
     urls: TaskUrls = None
 
+    def to_res_dict(self, return_file_urls: bool) -> dict:
+        remove_fields = ["deleted_model_ids"]
+        if not return_file_urls:
+            remove_fields.append("urls")
+
+        # noinspection PyTypeChecker
+        res = attr.asdict(
+            self, filter=lambda attrib, value: attrib.name not in remove_fields
+        )
+        if not return_file_urls:
+            res["urls"] = None
+
+        return res
+
     def __add__(self, other: "CleanupResult"):
         if not other:
             return self
@@ -66,40 +79,60 @@ class CleanupResult:
             updated_models=self.updated_models + other.updated_models,
             deleted_models=self.deleted_models + other.deleted_models,
             urls=self.urls + other.urls if self.urls else other.urls,
+            deleted_model_ids=self.deleted_model_ids | other.deleted_model_ids,
+        )
+
+    @staticmethod
+    def empty():
+        return CleanupResult(
+            updated_children=0,
+            updated_models=0,
+            deleted_models=0,
+            deleted_model_ids=set(),
         )
 
 
-def collect_plot_image_urls(company: str, task_or_model: str) -> Set[str]:
+def collect_plot_image_urls(
+    company: str, task_or_model: Union[str, Sequence[str]]
+) -> Set[str]:
     urls = set()
-    next_scroll_id = None
-    while True:
-        events, next_scroll_id = event_bll.get_plot_image_urls(
-            company_id=company, task_id=task_or_model, scroll_id=next_scroll_id
-        )
-        if not events:
-            break
-        for event in events:
-            event_urls = event.get(PlotFields.source_urls)
-            if event_urls:
-                urls.update(set(event_urls))
+    task_ids = task_or_model if isinstance(task_or_model, list) else [task_or_model]
+    for tasks in chunked_iter(task_ids, 100):
+        next_scroll_id = None
+        while True:
+            events, next_scroll_id = event_bll.get_plot_image_urls(
+                company_id=company, task_ids=tasks, scroll_id=next_scroll_id
+            )
+            if not events:
+                break
+            for event in events:
+                event_urls = event.get(PlotFields.source_urls)
+                if event_urls:
+                    urls.update(set(event_urls))
 
     return urls
 
 
-def collect_debug_image_urls(company: str, task_or_model: str) -> Set[str]:
+def collect_debug_image_urls(
+    company: str, task_or_model: Union[str, Sequence[str]]
+) -> Set[str]:
     """
     Return the set of unique image urls
     Uses DebugImagesIterator to make sure that we do not retrieve recycled urls
     """
-    after_key = None
     urls = set()
-    while True:
-        res, after_key = event_bll.get_debug_image_urls(
-            company_id=company, task_id=task_or_model, after_key=after_key,
-        )
-        urls.update(res)
-        if not after_key:
-            break
+    task_ids = task_or_model if isinstance(task_or_model, list) else [task_or_model]
+    for tasks in chunked_iter(task_ids, 100):
+        after_key = None
+        while True:
+            res, after_key = event_bll.get_debug_image_urls(
+                company_id=company,
+                task_ids=tasks,
+                after_key=after_key,
+            )
+            urls.update(res)
+            if not after_key:
+                break
 
     return urls
 
@@ -121,8 +154,12 @@ supported_storage_types.update(
 )
 
 
-def _schedule_for_delete(
-    company: str, user: str, task_id: str, urls: Set[str], can_delete_folders: bool,
+def schedule_for_delete(
+    company: str,
+    user: str,
+    task_id: str,
+    urls: Set[str],
+    can_delete_folders: bool,
 ) -> Set[str]:
     urls_per_storage = bucketize(
         urls,
@@ -184,15 +221,27 @@ def _schedule_for_delete(
     return processed_urls
 
 
+def delete_task_events_and_collect_urls(
+    company: str, task_ids: Sequence[str], wait_for_delete: bool, model=False
+) -> Set[str]:
+    event_urls = collect_debug_image_urls(company, task_ids) | collect_plot_image_urls(
+        company, task_ids
+    )
+
+    event_bll.delete_task_events(
+        company, task_ids, model=model, wait_for_delete=wait_for_delete
+    )
+
+    return event_urls
+
+
 def cleanup_task(
     company: str,
     user: str,
     task: Task,
     force: bool = False,
     update_children=True,
-    return_file_urls=False,
     delete_output_models=True,
-    delete_external_artifacts=True,
 ) -> CleanupResult:
     """
     Validate task deletion and delete/modify all its output.
@@ -203,88 +252,69 @@ def cleanup_task(
     published_models, draft_models, in_use_model_ids = verify_task_children_and_ouptuts(
         task, force
     )
-    delete_external_artifacts = delete_external_artifacts and config.get(
-        "services.async_urls_delete.enabled", True
-    )
-    event_urls, artifact_urls, model_urls = set(), set(), set()
-    if return_file_urls or delete_external_artifacts:
-        event_urls = collect_debug_image_urls(task.company, task.id)
-        event_urls.update(collect_plot_image_urls(task.company, task.id))
-        if task.execution and task.execution.artifacts:
-            artifact_urls = {
-                a.uri
-                for a in task.execution.artifacts.values()
-                if a.mode == ArtifactModes.output and a.uri
-            }
-        model_urls = {
-            m.uri for m in draft_models if m.uri and m.id not in in_use_model_ids
+    artifact_urls = (
+        {
+            a.uri
+            for a in task.execution.artifacts.values()
+            if a.mode == ArtifactModes.output and a.uri
         }
+        if task.execution and task.execution.artifacts
+        else {}
+    )
+    model_urls = {m.uri for m in draft_models if m.uri and m.id not in in_use_model_ids}
 
     deleted_task_id = f"{deleted_prefix}{task.id}"
     updated_children = 0
+    now = datetime.utcnow()
     if update_children:
-        updated_children = Task.objects(parent=task.id).update(parent=deleted_task_id)
+        updated_children = Task.objects(parent=task.id).update(
+            parent=deleted_task_id,
+            last_change=now,
+            last_changed_by=user,
+        )
 
     deleted_models = 0
     updated_models = 0
+    deleted_model_ids = set()
     for models, allow_delete in ((draft_models, True), (published_models, False)):
         if not models:
             continue
         if delete_output_models and allow_delete:
-            model_ids = set(m.id for m in models if m.id not in in_use_model_ids)
-            for m_id in model_ids:
-                if return_file_urls or delete_external_artifacts:
-                    event_urls.update(collect_debug_image_urls(task.company, m_id))
-                    event_urls.update(collect_plot_image_urls(task.company, m_id))
-                try:
-                    event_bll.delete_task_events(
-                        task.company,
-                        m_id,
-                        allow_locked=True,
-                        model=True,
-                        async_delete=async_events_delete,
-                    )
-                except errors.bad_request.InvalidModelId as ex:
-                    log.info(f"Error deleting events for the model {m_id}: {str(ex)}")
+            model_ids = list({m.id for m in models if m.id not in in_use_model_ids})
+            if model_ids:
+                deleted_models += Model.objects(id__in=model_ids).delete()
+                deleted_model_ids.update(model_ids)
 
-            deleted_models += Model.objects(id__in=list(model_ids)).delete()
             if in_use_model_ids:
-                Model.objects(id__in=list(in_use_model_ids)).update(unset__task=1)
+                Model.objects(id__in=list(in_use_model_ids)).update(
+                    unset__task=1,
+                    set__last_change=now,
+                    set__last_changed_by=user,
+                )
             continue
 
         if update_children:
             updated_models += Model.objects(id__in=[m.id for m in models]).update(
-                task=deleted_task_id
+                task=deleted_task_id,
+                last_change=now,
+                last_changed_by=user,
             )
         else:
-            Model.objects(id__in=[m.id for m in models]).update(unset__task=1)
-
-    event_bll.delete_task_events(
-        task.company, task.id, allow_locked=force, async_delete=async_events_delete
-    )
-
-    if delete_external_artifacts:
-        scheduled = _schedule_for_delete(
-            task_id=task.id,
-            company=company,
-            user=user,
-            urls=event_urls | model_urls | artifact_urls,
-            can_delete_folders=not in_use_model_ids and not published_models,
-        )
-        for urls in (event_urls, model_urls, artifact_urls):
-            urls.difference_update(scheduled)
+            Model.objects(id__in=[m.id for m in models]).update(
+                unset__task=1,
+                set__last_change=now,
+                set__last_changed_by=user,
+            )
 
     return CleanupResult(
         deleted_models=deleted_models,
         updated_children=updated_children,
         updated_models=updated_models,
         urls=TaskUrls(
-            event_urls=list(event_urls),
             artifact_urls=list(artifact_urls),
             model_urls=list(model_urls),
-        )
-        if return_file_urls
-        else None,
+        ),
+        deleted_model_ids=deleted_model_ids,
     )
 
 
@@ -304,7 +334,8 @@ def verify_task_children_and_ouptuts(
 
     model_fields = ["id", "ready", "uri"]
     published_models, draft_models = partition(
-        Model.objects(task=task.id).only(*model_fields), key=attrgetter("ready"),
+        Model.objects(task=task.id).only(*model_fields),
+        key=attrgetter("ready"),
     )
     if not force and published_models:
         raise errors.bad_request.TaskCannotBeDeleted(

apiserver/bll/task/task_operations.py

@@ -7,9 +7,10 @@ from apiserver.bll.task import (
     TaskBLL,
     validate_status_change,
     ChangeStatusRequest,
-    update_project_time,
 )
 from apiserver.bll.task.task_cleanup import cleanup_task, CleanupResult
+from apiserver.bll.task.utils import get_task_with_write_access
+from apiserver.bll.util import update_project_time
 from apiserver.config_repo import config
 from apiserver.database.model import EntityVisibility
 from apiserver.database.model.model import Model
@@ -22,84 +23,137 @@ from apiserver.database.model.task.task import (
     ArtifactModes,
     Execution,
     DEFAULT_LAST_ITERATION,
+    TaskType,
 )
 from apiserver.database.utils import get_options
+from apiserver.service_repo.auth import Identity
 from apiserver.utilities.dicts import nested_set
 
 log = config.logger(__file__)
 queue_bll = QueueBLL()
 
 
+def _get_pipeline_steps_for_controller_task(
+    task: Task, company_id: str, only: Sequence[str] = None
+) -> Sequence[Task]:
+    if not task or task.type != TaskType.controller:
+        return []
+
+    query = Task.objects(company=company_id, parent=task.id)
+    if only:
+        query = query.only(*only)
+
+    return list(query)
+
+
 def archive_task(
     task: Union[str, Task],
     company_id: str,
-    user_id: str,
+    identity: Identity,
     status_message: str,
     status_reason: str,
+    include_pipeline_steps: bool,
 ) -> int:
     """
     Deque and archive task
     Return 1 if successful
     """
+    user_id = identity.user
+    fields = (
+        "id",
+        "company",
+        "execution",
+        "status",
+        "project",
+        "system_tags",
+        "enqueue_status",
+        "type",
+    )
     if isinstance(task, str):
-        task = TaskBLL.get_task_with_access(
+        task = get_task_with_write_access(
             task,
             company_id=company_id,
-            only=(
-                "id",
-                "company",
-                "execution",
-                "status",
-                "project",
-                "system_tags",
-                "enqueue_status",
-            ),
-            requires_write_access=True,
+            identity=identity,
+            only=fields,
         )
-    try:
-        TaskBLL.dequeue_and_change_status(
-            task,
-            company_id=company_id,
-            user_id=user_id,
+
+    def archive_task_core(task_: Task) -> int:
+        try:
+            TaskBLL.dequeue_and_change_status(
+                task_,
+                company_id=company_id,
+                user_id=user_id,
+                status_message=status_message,
+                status_reason=status_reason,
+                remove_from_all_queues=True,
+                new_status_for_aborted_task=TaskStatus.stopped,
+            )
+        except APIError:
+            # dequeue may fail if the task was not enqueued
+            pass
+
+        return task_.update(
             status_message=status_message,
             status_reason=status_reason,
-            remove_from_all_queues=True,
+            add_to_set__system_tags=EntityVisibility.archived.value,
+            last_change=datetime.utcnow(),
+            last_changed_by=user_id,
         )
-    except APIError:
-        # dequeue may fail if the task was not enqueued
-        pass
 
-    return task.update(
-        status_message=status_message,
-        status_reason=status_reason,
-        add_to_set__system_tags=EntityVisibility.archived.value,
-        last_change=datetime.utcnow(),
-        last_changed_by=user_id,
-    )
+    if include_pipeline_steps and (
+        step_tasks := _get_pipeline_steps_for_controller_task(
+            task, company_id, only=fields
+        )
+    ):
+        for step in step_tasks:
+            archive_task_core(step)
+
+    return archive_task_core(task)
 
 
 def unarchive_task(
-    task: str, company_id: str, user_id: str, status_message: str, status_reason: str,
+    task_id: str,
+    company_id: str,
+    identity: Identity,
+    status_message: str,
+    status_reason: str,
+    include_pipeline_steps: bool,
 ) -> int:
     """
     Unarchive task. Return 1 if successful
     """
-    task = TaskBLL.get_task_with_access(
-        task, company_id=company_id, only=("id",), requires_write_access=True,
-    )
-    return task.update(
-        status_message=status_message,
-        status_reason=status_reason,
-        pull__system_tags=EntityVisibility.archived.value,
-        last_change=datetime.utcnow(),
-        last_changed_by=user_id,
+    fields = ("id", "type")
+    task = get_task_with_write_access(
+        task_id,
+        company_id=company_id,
+        identity=identity,
+        only=fields,
     )
 
+    def unarchive_task_core(task_: Task) -> int:
+        return task_.update(
+            status_message=status_message,
+            status_reason=status_reason,
+            pull__system_tags=EntityVisibility.archived.value,
+            last_change=datetime.utcnow(),
+            last_changed_by=identity.user,
+        )
+
+    if include_pipeline_steps and (
+        step_tasks := _get_pipeline_steps_for_controller_task(
+            task, company_id, only=fields
+        )
+    ):
+        for step in step_tasks:
+            unarchive_task_core(step)
+
+    return unarchive_task_core(task)
+
 
 def dequeue_task(
     task_id: str,
     company_id: str,
-    user_id: str,
+    identity: Identity,
     status_message: str,
     status_reason: str,
     remove_from_all_queues: bool = False,
@@ -112,7 +166,19 @@ def dequeue_task(
     task = Task.get(
         id=task_id,
         company=company_id,
-        _only=(
+        _only=("id",),
+        include_public=True,
+    )
+    if not task:
+        TaskBLL.remove_task_from_all_queues(company_id, task_id=task_id)
+        return 1, {"updated": 0}
+
+    user_id = identity.user
+    task = get_task_with_write_access(
+        task_id,
+        company_id=company_id,
+        identity=identity,
+        only=(
             "id",
             "company",
             "execution",
@@ -120,11 +186,7 @@ def dequeue_task(
             "project",
             "enqueue_status",
         ),
-        include_public=True,
     )
-    if not task:
-        TaskBLL.remove_task_from_all_queues(company_id, task_id=task_id)
-        return 1, {"updated": 0}
 
     res = TaskBLL.dequeue_and_change_status(
         task,
@@ -141,19 +203,32 @@ def dequeue_task(
 def enqueue_task(
     task_id: str,
     company_id: str,
-    user_id: str,
+    identity: Identity,
     queue_id: str,
     status_message: str,
     status_reason: str,
     queue_name: str = None,
     validate: bool = False,
     force: bool = False,
+    update_execution_queue: bool = True,
 ) -> Tuple[int, dict]:
     if queue_id and queue_name:
         raise errors.bad_request.ValidationError(
             "Either queue id or queue name should be provided"
         )
 
+    task = get_task_with_write_access(
+        task_id=task_id, company_id=company_id, identity=identity
+    )
+    if not update_execution_queue:
+        if not (
+            task.status == TaskStatus.queued and task.execution and task.execution.queue
+        ):
+            raise errors.bad_request.ValidationError(
+                "Cannot skip setting execution queue for a task "
+                "that is not enqueued or does not have execution queue set"
+            )
+
     if queue_name:
         queue = queue_bll.get_by_name(
             company_id=company_id, queue_name=queue_name, only=("id",)
@@ -166,23 +241,21 @@ def enqueue_task(
         # try to get default queue
         queue_id = queue_bll.get_default(company_id).id
 
-    query = dict(id=task_id, company=company_id)
-    task = Task.get_for_writing(**query)
-    if not task:
-        raise errors.bad_request.InvalidTaskId(**query)
-
+    user_id = identity.user
     if validate:
         TaskBLL.validate(task)
 
+    before_enqueue_status = task.status
+    if task.status == TaskStatus.queued and task.enqueue_status:
+        before_enqueue_status = task.enqueue_status
     res = ChangeStatusRequest(
         task=task,
         new_status=TaskStatus.queued,
         status_reason=status_reason,
         status_message=status_message,
-        allow_same_state_transition=False,
         force=force,
         user_id=user_id,
-    ).execute(enqueue_status=task.status)
+    ).execute(enqueue_status=before_enqueue_status)
 
     try:
         queue_bll.add_task(company_id=company_id, queue_id=queue_id, task_id=task.id)
@@ -199,12 +272,19 @@ def enqueue_task(
         raise
 
     # set the current queue ID in the task
-    if task.execution:
-        Task.objects(**query).update(execution__queue=queue_id, multi=False)
-    else:
-        Task.objects(**query).update(execution=Execution(queue=queue_id), multi=False)
+    if update_execution_queue:
+        if task.execution:
+            Task.objects(id=task_id).update(execution__queue=queue_id, multi=False)
+        else:
+            Task.objects(id=task_id).update(
+                execution=Execution(queue=queue_id), multi=False
+            )
+        nested_set(res, ("fields", "execution.queue"), queue_id)
 
-    nested_set(res, ("fields", "execution.queue"), queue_id)
+    # make sure that the task is not queued in any other queue
+    TaskBLL.remove_task_from_all_queues(
+        company_id=company_id, task_id=task_id, exclude=queue_id
+    )
     return 1, res
 
 
@@ -235,18 +315,16 @@ def move_tasks_to_trash(tasks: Sequence[str]) -> int:
 def delete_task(
     task_id: str,
     company_id: str,
-    user_id: str,
+    identity: Identity,
     move_to_trash: bool,
     force: bool,
-    return_file_urls: bool,
     delete_output_models: bool,
     status_message: str,
     status_reason: str,
-    delete_external_artifacts: bool,
+    include_pipeline_steps: bool,
 ) -> Tuple[int, Task, CleanupResult]:
-    task = TaskBLL.get_task_with_access(
-        task_id, company_id=company_id, requires_write_access=True
-    )
+    user_id = identity.user
+    task = get_task_with_write_access(task_id, company_id=company_id, identity=identity)
 
     if (
         task.status != TaskStatus.created
@@ -260,36 +338,50 @@ def delete_task(
             current=task.status,
         )
 
-    try:
-        TaskBLL.dequeue_and_change_status(
-            task,
-            company_id=company_id,
-            user_id=user_id,
-            status_message=status_message,
-            status_reason=status_reason,
-            remove_from_all_queues=True,
+    def delete_task_core(task_: Task, force_: bool) -> CleanupResult:
+        try:
+            TaskBLL.dequeue_and_change_status(
+                task_,
+                company_id=company_id,
+                user_id=user_id,
+                status_message=status_message,
+                status_reason=status_reason,
+                remove_from_all_queues=True,
+            )
+        except APIError:
+            # dequeue may fail if the task was not enqueued
+            pass
+
+        res = cleanup_task(
+            company=company_id,
+            user=user_id,
+            task=task_,
+            force=force_,
+            delete_output_models=delete_output_models,
         )
-    except APIError:
-        # dequeue may fail if the task was not enqueued
-        pass
 
-    cleanup_res = cleanup_task(
-        company=company_id,
-        user=user_id,
-        task=task,
-        force=force,
-        return_file_urls=return_file_urls,
-        delete_output_models=delete_output_models,
-        delete_external_artifacts=delete_external_artifacts,
-    )
+        if move_to_trash:
+            # make sure that whatever changes were done to the task are saved
+            # the task itself will be deleted later in the move_tasks_to_trash operation
+            task_.last_update = datetime.utcnow()
+            task_.save()
+        else:
+            task_.delete()
 
+        return res
+
+    task_ids = [task.id]
+    cleanup_res = CleanupResult.empty()
+    if include_pipeline_steps and (
+        step_tasks := _get_pipeline_steps_for_controller_task(task, company_id)
+    ):
+        for step in step_tasks:
+            cleanup_res += delete_task_core(step, True)
+            task_ids.append(step.id)
+
+    cleanup_res = delete_task_core(task, force)
     if move_to_trash:
-        # make sure that whatever changes were done to the task are saved
-        # the task itself will be deleted later in the move_tasks_to_trash operation
-        task.last_update = datetime.utcnow()
-        task.save()
-    else:
-        task.delete()
+        move_tasks_to_trash(task_ids)
 
     update_project_time(task.project)
     return 1, task, cleanup_res
@@ -298,16 +390,13 @@ def delete_task(
 def reset_task(
     task_id: str,
     company_id: str,
-    user_id: str,
+    identity: Identity,
     force: bool,
-    return_file_urls: bool,
     delete_output_models: bool,
     clear_all: bool,
-    delete_external_artifacts: bool,
 ) -> Tuple[dict, CleanupResult, dict]:
-    task = TaskBLL.get_task_with_access(
-        task_id, company_id=company_id, requires_write_access=True
-    )
+    user_id = identity.user
+    task = get_task_with_write_access(task_id, company_id=company_id, identity=identity)
 
     if not force and task.status == TaskStatus.published:
         raise errors.bad_request.InvalidTaskStatus(task_id=task.id, status=task.status)
@@ -316,7 +405,9 @@ def reset_task(
     updates = {}
 
     try:
-        dequeued = TaskBLL.dequeue(task, company_id, silent_fail=True)
+        dequeued = TaskBLL.dequeue(
+            task, company_id=company_id, user_id=user_id, silent_fail=True
+        )
     except APIError:
         # dequeue may fail if the task was not enqueued
         pass
@@ -329,9 +420,7 @@ def reset_task(
         task=task,
         force=force,
         update_children=False,
-        return_file_urls=return_file_urls,
         delete_output_models=delete_output_models,
-        delete_external_artifacts=delete_external_artifacts,
     )
 
     updates.update(
@@ -345,11 +434,17 @@ def reset_task(
         unset__output__error=1,
         unset__last_worker=1,
         unset__last_worker_report=1,
+        unset__started=1,
+        unset__completed=1,
+        unset__published=1,
+        unset__active_duration=1,
+        unset__enqueue_status=1,
     )
 
     if clear_all:
         updates.update(
-            set__execution=Execution(), unset__script=1,
+            set__execution=Execution(),
+            unset__script=1,
         )
     else:
         updates.update(unset__execution__queue=1)
@@ -370,11 +465,6 @@ def reset_task(
         status_message="reset",
         user_id=user_id,
     ).execute(
-        started=None,
-        completed=None,
-        published=None,
-        active_duration=None,
-        enqueue_status=None,
         **updates,
     )
 
@@ -384,15 +474,14 @@ def reset_task(
 def publish_task(
     task_id: str,
     company_id: str,
-    user_id: str,
+    identity: Identity,
     force: bool,
-    publish_model_func: Callable[[str, str, str], Any] = None,
+    publish_model_func: Callable[[str, str, Identity], Any] = None,
     status_message: str = "",
     status_reason: str = "",
 ) -> dict:
-    task = TaskBLL.get_task_with_access(
-        task_id, company_id=company_id, requires_write_access=True
-    )
+    user_id = identity.user
+    task = get_task_with_write_access(task_id, company_id=company_id, identity=identity)
     if not force:
         validate_status_change(task.status, TaskStatus.published)
 
@@ -414,7 +503,7 @@ def publish_task(
                 .first()
             )
             if model and not model.ready:
-                publish_model_func(model.id, company_id, user_id)
+                publish_model_func(model.id, company_id, identity)
 
         # set task status to published, and update (or set) it's new output (view and models)
         return ChangeStatusRequest(
@@ -438,10 +527,11 @@ def publish_task(
 def stop_task(
     task_id: str,
     company_id: str,
-    user_id: str,
+    identity: Identity,
     user_name: str,
     status_reason: str,
     force: bool,
+    include_pipeline_steps: bool,
 ) -> dict:
     """
     Stop a running task. Requires task status 'in_progress' and
@@ -451,20 +541,22 @@ def stop_task(
     is set to 'stopping' to allow the worker to stop the task and report by itself
     :return: updated task fields
     """
-
-    task = TaskBLL.get_task_with_access(
+    user_id = identity.user
+    fields = (
+        "status",
+        "project",
+        "tags",
+        "system_tags",
+        "last_worker",
+        "last_update",
+        "execution.queue",
+        "type",
+    )
+    task = get_task_with_write_access(
         task_id,
         company_id=company_id,
-        only=(
-            "status",
-            "project",
-            "tags",
-            "system_tags",
-            "last_worker",
-            "last_update",
-            "execution.queue",
-        ),
-        requires_write_access=True,
+        identity=identity,
+        only=fields,
     )
 
     def is_run_by_worker(t: Task) -> bool:
@@ -476,32 +568,45 @@ def stop_task(
             and (datetime.utcnow() - t.last_update).total_seconds() < update_timeout
         )
 
-    is_queued = task.status == TaskStatus.queued
-    set_stopped = (
-        is_queued
-        or TaskSystemTags.development in task.system_tags
-        or not is_run_by_worker(task)
-    )
+    def stop_task_core(task_: Task, force_: bool):
+        is_queued = task_.status == TaskStatus.queued
+        set_stopped = (
+            is_queued
+            or TaskSystemTags.development in task_.system_tags
+            or not is_run_by_worker(task_)
+        )
 
-    if set_stopped:
-        if is_queued:
-            try:
-                TaskBLL.dequeue(task, company_id=company_id, silent_fail=True)
-            except APIError:
-                # dequeue may fail if the task was not enqueued
-                pass
+        if set_stopped:
+            if is_queued:
+                try:
+                    TaskBLL.dequeue(
+                        task_, company_id=company_id, user_id=user_id, silent_fail=True
+                    )
+                except APIError:
+                    # dequeue may fail if the task was not enqueued
+                    pass
 
-        new_status = TaskStatus.stopped
-        status_message = f"Stopped by {user_name}"
-    else:
-        new_status = task.status
-        status_message = TaskStatusMessage.stopping
+            new_status = TaskStatus.stopped
+            status_message = f"Stopped by {user_name}"
+        else:
+            new_status = task_.status
+            status_message = TaskStatusMessage.stopping
 
-    return ChangeStatusRequest(
-        task=task,
-        new_status=new_status,
-        status_reason=status_reason,
-        status_message=status_message,
-        force=force,
-        user_id=user_id,
-    ).execute()
+        return ChangeStatusRequest(
+            task=task_,
+            new_status=new_status,
+            status_reason=status_reason,
+            status_message=status_message,
+            force=force_,
+            user_id=user_id,
+        ).execute()
+
+    if include_pipeline_steps and (
+        step_tasks := _get_pipeline_steps_for_controller_task(
+            task, company_id, only=fields
+        )
+    ):
+        for step in step_tasks:
+            stop_task_core(step, True)
+
+    return stop_task_core(task, force)

apiserver/bll/task/utils.py

@@ -1,16 +1,19 @@
 from datetime import datetime
-from typing import Sequence, Union
+from typing import Sequence
 
 import attr
 import six
+from mongoengine import Q
+from mongoengine.base import UPDATE_OPERATORS
 
 from apiserver.apierrors import errors
+from apiserver.bll.util import update_project_time
 from apiserver.config_repo import config
 from apiserver.database.errors import translate_errors_context
 from apiserver.database.model.model import Model
-from apiserver.database.model.project import Project
 from apiserver.database.model.task.task import Task, TaskStatus, TaskSystemTags
 from apiserver.database.utils import get_options
+from apiserver.service_repo.auth import Identity
 from apiserver.utilities.attrs import typed_attrs
 
 valid_statuses = get_options(TaskStatus)
@@ -76,8 +79,16 @@ class ChangeStatusRequest(object):
 
         update_project_time(project_id)
 
-        # make sure that _raw_ queries are not returned back to the client
-        fields.pop("__raw__", None)
+        def is_mongo_operator(field: str) -> bool:
+            head, _, tail = field.partition("__")
+            return tail and (head in UPDATE_OPERATORS)
+
+        # make sure to not return _raw_ queries or any of the update operators
+        fields = {
+            key: value
+            for key, value in fields.items()
+            if not (key == "__raw__" or is_mongo_operator(key))
+        }
 
         return dict(updated=updated, fields=fields)
 
@@ -133,7 +144,12 @@ state_machine = {
         TaskStatus.publishing,
         TaskStatus.stopped,
     },
-    TaskStatus.failed: {TaskStatus.created, TaskStatus.stopped, TaskStatus.published},
+    TaskStatus.failed: {
+        TaskStatus.created,
+        TaskStatus.stopped,
+        TaskStatus.published,
+        TaskStatus.queued,
+    },
     TaskStatus.publishing: {TaskStatus.published},
     TaskStatus.published: set(),
     TaskStatus.completed: {
@@ -158,25 +174,78 @@ def get_possible_status_changes(current_status):
     return possible
 
 
-def update_project_time(project_ids: Union[str, Sequence[str]]):
-    if not project_ids:
-        return
+def get_many_tasks_for_writing(
+    company_id: str,
+    identity: Identity,
+    query: Q = None,
+    only: Sequence = None,
+    throw_on_forbidden: bool = True,
+) -> Sequence[Task]:
+    if only:
+        missing = [f for f in ("company",) if f not in only]
+        if missing:
+            only = [*only, *missing]
 
-    if isinstance(project_ids, str):
-        project_ids = [project_ids]
+    result = list(
+        Task.get_many(
+            company=company_id,
+            query=query,
+            override_projection=only,
+            allow_public=True,
+            return_dicts=False,
+        )
+    )
 
-    return Project.objects(id__in=project_ids).update(last_update=datetime.utcnow())
+    if not company_id:
+        return result
+
+    forbidden_tasks = {task.id for task in result if not task.company}
+    if forbidden_tasks:
+        if throw_on_forbidden:
+            raise errors.forbidden.NoWritePermission(
+                f"cannot modify public task(s), ids={tuple(forbidden_tasks)}"
+            )
+        result = [task for task in result if task.id not in forbidden_tasks]
+
+    return result
+
+
+def get_task_with_write_access(
+    task_id: str,
+    company_id: str,
+    identity: Identity,
+    only=None,
+) -> Task:
+    """
+    Gets a task that has a required write access
+    :except errors.bad_request.InvalidTaskId: if the task is not found
+    :except errors.forbidden.NoWritePermission: if write_access was required and the task cannot be modified
+    """
+    query = dict(id=task_id, company=company_id)
+
+    task = Task.get_for_writing(_only=only, **query)
+    if not task:
+        raise errors.bad_request.InvalidTaskId(**query)
+
+    return task
 
 
 def get_task_for_update(
-    company_id: str, task_id: str, allow_all_statuses: bool = False, force: bool = False
+    company_id: str,
+    task_id: str,
+    identity: Identity,
+    allow_all_statuses: bool = False,
+    force: bool = False,
 ) -> Task:
     """
     Loads only task id and return the task only if it is updatable (status == 'created')
     """
-    task = Task.get_for_writing(company=company_id, id=task_id, _only=("id", "status"))
-    if not task:
-        raise errors.bad_request.InvalidTaskId(id=task_id)
+    task = get_task_with_write_access(
+        task_id=task_id,
+        company_id=company_id,
+        only=("id", "status"),
+        identity=identity,
+    )
 
     if allow_all_statuses:
         return task
@@ -222,13 +291,62 @@ def get_last_metric_updates(
 
     new_metrics = []
 
+    def add_last_metric_mean_update(
+        metric_path: str,
+        metric_count: int,
+        metric_total: float,
+    ):
+        """
+        Update new mean field based on the value in db and new data
+        The count field is updated here too and not with inc__ so that
+        it will not get updated in the db earlier than the corresponding mean
+        """
+        metric_path = metric_path.replace("__", ".")
+        mean_value_field = f"{metric_path}.mean_value"
+        count_field = f"{metric_path}.count"
+        raw_updates[mean_value_field] = {
+            "$round": [
+                {
+                    "$divide": [
+                        {
+                            "$add": [
+                                {
+                                    "$multiply": [
+                                        {"$ifNull": [f"${mean_value_field}", 0]},
+                                        {"$ifNull": [f"${count_field}", 0]},
+                                    ]
+                                },
+                                metric_total,
+                            ]
+                        },
+                        {
+                            "$add": [
+                                {"$ifNull": [f"${count_field}", 0]},
+                                metric_count,
+                            ]
+                        },
+                    ]
+                },
+                2,
+            ]
+        }
+        raw_updates[count_field] = {
+            "$add": [
+                {"$ifNull": [f"${count_field}", 0]},
+                metric_count,
+            ]
+        }
+
     def add_last_metric_conditional_update(
-        metric_path: str, metric_value, iter_value: int, is_min: bool
+        metric_path: str, metric_value, iter_value: int, is_min: bool, is_first: bool
     ):
         """
         Build an aggregation for an atomic update of the min or max value and the corresponding iteration
         """
-        if is_min:
+        if is_first:
+            field_prefix = "first"
+            op = None
+        elif is_min:
             field_prefix = "min"
             op = "$gt"
         else:
@@ -236,18 +354,23 @@ def get_last_metric_updates(
             op = "$lt"
 
         value_field = f"{metric_path}__{field_prefix}_value".replace("__", ".")
-        condition = {
-            "$or": [
-                {"$lte": [f"${value_field}", None]},
-                {op: [f"${value_field}", metric_value]},
-            ]
-        }
+        exists = {"$lte": [f"${value_field}", None]}
+        if op:
+            condition = {
+                "$or": [
+                    exists,
+                    {op: [f"${value_field}", metric_value]},
+                ]
+            }
+        else:
+            condition = exists
+
         raw_updates[value_field] = {
             "$cond": [condition, metric_value, f"${value_field}"]
         }
 
-        value_iteration_field = f"{metric_path}__{field_prefix}_value_iteration".replace(
-            "__", "."
+        value_iteration_field = (
+            f"{metric_path}__{field_prefix}_value_iteration".replace("__", ".")
         )
         raw_updates[value_iteration_field] = {
             "$cond": [condition, iter_value, f"${value_iteration_field}"]
@@ -264,15 +387,25 @@ def get_last_metric_updates(
             new_metrics.append(metric)
             path = f"last_metrics__{metric_key}__{variant_key}"
             for key, value in variant_data.items():
-                if key in ("min_value", "max_value"):
+                if key in ("min_value", "max_value", "first_value"):
                     add_last_metric_conditional_update(
                         metric_path=path,
                         metric_value=value,
                         iter_value=variant_data.get(f"{key}_iter", 0),
                         is_min=(key == "min_value"),
+                        is_first=(key == "first_value"),
                     )
                 elif key in ("metric", "variant", "value"):
                     extra_updates[f"set__{path}__{key}"] = value
 
+            count = variant_data.get("count")
+            total = variant_data.get("total")
+            if count is not None and total is not None:
+                add_last_metric_mean_update(
+                    metric_path=path,
+                    metric_count=count,
+                    metric_total=total,
+                )
+
     if new_metrics:
         extra_updates["add_to_set__unique_metrics"] = new_metrics

apiserver/bll/user/__init__.py

@@ -2,6 +2,7 @@ from datetime import datetime
 
 from apiserver.apierrors import errors
 from apiserver.apimodels.users import CreateRequest
+from apiserver.config.info import get_version
 from apiserver.database.errors import translate_errors_context
 from apiserver.database.model.user import User
 
@@ -14,7 +15,11 @@ class UserBLL:
             if user_id and User.objects(id=user_id).only("id"):
                 raise errors.bad_request.UserIdExists(id=user_id)
 
-            user = User(**request.to_struct(), created=datetime.utcnow())
+            user = User(
+                **request.to_struct(),
+                created=datetime.utcnow(),
+                created_in_version=get_version(),
+            )
             user.save(force_insert=True)
 
     @staticmethod

apiserver/bll/util.py

@@ -1,6 +1,7 @@
 import functools
 import itertools
 from concurrent.futures.thread import ThreadPoolExecutor
+from datetime import datetime
 from typing import (
     Optional,
     Callable,
@@ -8,11 +9,13 @@ from typing import (
     Tuple,
     Sequence,
     TypeVar,
+    Union,
 )
 
 from boltons import iterutils
 
 from apiserver.apierrors import APIError
+from apiserver.database.model.project import Project
 from apiserver.database.model.settings import Settings
 
 
@@ -77,3 +80,13 @@ def run_batch_operation(
                 }
             )
     return results, failures
+
+
+def update_project_time(project_ids: Union[str, Sequence[str]]):
+    if not project_ids:
+        return
+
+    if isinstance(project_ids, str):
+        project_ids = [project_ids]
+
+    return Project.objects(id__in=project_ids).update(last_update=datetime.utcnow())

apiserver/bll/workers/__init__.py

@@ -1,17 +1,18 @@
 import itertools
+import re
 from datetime import datetime, timedelta
 from time import time
 from typing import Sequence, Set, Optional
 
 import attr
 import elasticsearch.helpers
-from boltons.iterutils import partition
+from boltons.iterutils import partition, chunked_iter
+from pyhocon import ConfigTree
 
 from apiserver.es_factory import es_factory
 from apiserver.apierrors import APIError
 from apiserver.apierrors.errors import bad_request, server_error
 from apiserver.apimodels.workers import (
-    DEFAULT_TIMEOUT,
     IdNameEntry,
     WorkerEntry,
     StatusReportRequest,
@@ -27,15 +28,18 @@ from apiserver.database.model.project import Project
 from apiserver.database.model.queue import Queue
 from apiserver.database.model.task.task import Task
 from apiserver.redis_manager import redman
-from apiserver.tools import safe_get
+from apiserver.utilities.dicts import nested_get
 from .stats import WorkerStats
 
 log = config.logger(__file__)
 
 
 class WorkerBLL:
+    _key_regex_trans = str.maketrans({"*": ".*", "?": ".?"})
+
     def __init__(self, es=None, redis=None):
         self.es_client = es or es_factory.connect("workers")
+        self.config = config.get("services.workers", ConfigTree())
         self.redis = redis or redman.connection("workers")
         self._stats = WorkerStats(self.es_client)
 
@@ -68,7 +72,7 @@ class WorkerBLL:
         """
         key = WorkerBLL._get_worker_key(company_id, user_id, worker)
 
-        timeout = timeout or DEFAULT_TIMEOUT
+        timeout = timeout or int(self.config.get("default_worker_timeout_sec", 10 * 60))
         queues = queues or []
 
         with translate_errors_context():
@@ -141,8 +145,6 @@ class WorkerBLL:
 
         try:
             entry.ip = ip
-            now = datetime.utcnow()
-            entry.last_activity_time = now
 
             if tags is not None:
                 entry.tags = tags
@@ -150,15 +152,16 @@ class WorkerBLL:
                 entry.system_tags = system_tags
 
             if report.machine_stats:
-                self._log_stats_to_es(
+                self.log_stats_to_es(
                     company_id=company_id,
-                    company_name=entry.company.name,
-                    worker=entry.key,
+                    worker_id=report.worker,
                     timestamp=report.timestamp,
                     task=report.task,
                     machine_stats=report.machine_stats,
                 )
 
+            now = datetime.utcnow()
+            entry.last_activity_time = now
             entry.queue = report.queue
 
             if report.queues:
@@ -175,6 +178,7 @@ class WorkerBLL:
                         last_worker_report=now,
                         last_update=now,
                         last_change=now,
+                        last_changed_by=user_id,
                     )
                     # modify(new=True, ...) returns the modified object
                     task = Task.objects(**query).modify(new=True, **update)
@@ -206,15 +210,25 @@ class WorkerBLL:
         last_seen: Optional[int] = None,
         tags: Sequence[str] = None,
         system_tags: Sequence[str] = None,
+        worker_pattern: str = None,
     ):
         if not last_seen:
             return len(
-                self._get_keys(company_id, user_tags=tags, system_tags=system_tags)
+                self._get_keys(
+                    company_id,
+                    user_tags=tags,
+                    system_tags=system_tags,
+                    worker_pattern=worker_pattern,
+                )
             )
 
         return len(
             self.get_all(
-                company_id, last_seen=last_seen, tags=tags, system_tags=system_tags
+                company_id,
+                last_seen=last_seen,
+                tags=tags,
+                system_tags=system_tags,
+                worker_pattern=worker_pattern,
             )
         )
 
@@ -224,6 +238,7 @@ class WorkerBLL:
         last_seen: Optional[int] = None,
         tags: Sequence[str] = None,
         system_tags: Sequence[str] = None,
+        worker_pattern: str = None,
     ) -> Sequence[WorkerEntry]:
         """
         Get all the company workers that were active during the last_seen period
@@ -232,7 +247,12 @@ class WorkerBLL:
         :return:
         """
         try:
-            workers = self._get(company_id, user_tags=tags, system_tags=system_tags)
+            workers = self._get(
+                company_id,
+                user_tags=tags,
+                system_tags=system_tags,
+                worker_pattern=worker_pattern,
+            )
         except Exception as e:
             raise server_error.DataError("failed loading worker entries", err=e.args[0])
 
@@ -252,19 +272,18 @@ class WorkerBLL:
         last_seen: int,
         tags: Sequence[str] = None,
         system_tags: Sequence[str] = None,
+        worker_pattern: str = None,
     ) -> Sequence[WorkerResponseEntry]:
-
-        helpers = list(
-            map(
-                WorkerConversionHelper.from_worker_entry,
-                self.get_all(
-                    company_id=company_id,
-                    last_seen=last_seen,
-                    tags=tags,
-                    system_tags=system_tags,
-                ),
+        helpers = [
+            WorkerConversionHelper.from_worker_entry(entry)
+            for entry in self.get_all(
+                company_id=company_id,
+                last_seen=last_seen,
+                tags=tags,
+                system_tags=system_tags,
+                worker_pattern=worker_pattern,
             )
-        )
+        ]
 
         task_ids = set(filter(None, (helper.task_id for helper in helpers)))
         all_queues = set(
@@ -283,14 +302,12 @@ class WorkerBLL:
                     }
                 },
             ]
-            queues_info = {
-                res["_id"]: res for res in Queue.objects.aggregate(projection)
-            }
+            queues_info = {res["_id"]: res for res in Queue.aggregate(projection)}
             task_ids = task_ids.union(
                 filter(
                     None,
                     (
-                        safe_get(info, "next_entry/task")
+                        nested_get(info, ("next_entry", "task"))
                         for info in queues_info.values()
                     ),
                 )
@@ -314,7 +331,7 @@ class WorkerBLL:
                     continue
                 entry.name = info.get("name", None)
                 entry.num_tasks = info.get("num_entries", 0)
-                task_id = safe_get(info, "next_entry/task")
+                task_id = nested_get(info, ("next_entry", "task"))
                 if task_id:
                     task = tasks_info.get(task_id, None)
                     entry.next_task = IdNameEntry(
@@ -324,7 +341,7 @@ class WorkerBLL:
         for helper in helpers:
             worker = helper.worker
             if helper.task_id:
-                task = tasks_info.get(helper.task_id, None)
+                task: Task = tasks_info.get(helper.task_id, None)
                 if task:
                     worker.task.running_time = (task.active_duration or 0) * 1000
                     worker.task.last_iteration = task.last_iteration
@@ -420,16 +437,25 @@ class WorkerBLL:
         user: str = "*",
         user_tags: Sequence[str] = None,
         system_tags: Sequence[str] = None,
+        worker_pattern: str = None,
     ) -> Sequence[bytes]:
         if not (user_tags or system_tags):
-            match = self._get_worker_key(company, user, "*")
+            match = self._get_worker_key(company, user, worker_pattern or "*")
             return list(self.redis.scan_iter(match))
 
-        def filter_by_user(in_keys: Set[bytes]) -> Set[bytes]:
-            if user == "*":
-                return in_keys
-            user_bytes = user.encode()
-            return {k for k in in_keys if user_bytes in k}
+        def filter_by_user_and_pattern(in_keys: Set[bytes]) -> Set[bytes]:
+            if user != "*":
+                user_bytes = user.encode()
+                in_keys = {k for k in in_keys if user_bytes in k}
+
+            if worker_pattern:
+                worker_pattern_bytes = (
+                    f"{worker_pattern.translate(self._key_regex_trans)}$".encode()
+                )
+                regex = re.compile(worker_pattern_bytes)
+                in_keys = {k for k in in_keys if regex.search(k)}
+
+            return in_keys
 
         worker_keys = set()
         for tags, tags_field in (
@@ -452,7 +478,7 @@ class WorkerBLL:
                     )
                     tagged_workers.update(self.redis.zrange(tagged_workers_key, 0, -1))
 
-                tagged_workers = filter_by_user(tagged_workers)
+                tagged_workers = filter_by_user_and_pattern(tagged_workers)
                 worker_keys = (
                     worker_keys.intersection(tagged_workers)
                     if worker_keys
@@ -466,7 +492,7 @@ class WorkerBLL:
                     all_workers_key = self._get_all_workers_key(company)
                     self.redis.zremrangebyscore(all_workers_key, min=0, max=timestamp)
                     worker_keys.update(self.redis.zrange(all_workers_key, 0, -1))
-                    worker_keys = filter_by_user(worker_keys)
+                    worker_keys = filter_by_user_and_pattern(worker_keys)
                     if not worker_keys:
                         return []
 
@@ -491,16 +517,24 @@ class WorkerBLL:
         user: str = "*",
         user_tags: Sequence[str] = None,
         system_tags: Sequence[str] = None,
+        worker_pattern: str = None,
     ) -> Sequence[WorkerEntry]:
         """Get worker entries matching the company and user, worker patterns"""
 
         entries = []
-        for key in self._get_keys(
-            company, user=user, user_tags=user_tags, system_tags=system_tags
+        for keys in chunked_iter(
+            self._get_keys(
+                company,
+                user=user,
+                user_tags=user_tags,
+                system_tags=system_tags,
+                worker_pattern=worker_pattern,
+            ),
+            1000,
         ):
-            data = self.redis.get(key)
+            data = self.redis.mget(keys)
             if data:
-                entries.append(WorkerEntry.from_json(data))
+                entries.extend(WorkerEntry.from_json(d) for d in data if d)
 
         return entries
 
@@ -509,18 +543,17 @@ class WorkerBLL:
         """Get the index name suffix for storing current month data"""
         return datetime.utcnow().strftime("%Y-%m")
 
-    def _log_stats_to_es(
+    def log_stats_to_es(
         self,
         company_id: str,
-        company_name: str,
-        worker: str,
+        worker_id: str,
         timestamp: int,
         task: str,
         machine_stats: MachineStats,
-    ) -> bool:
+    ) -> int:
         """
         Actually writing the worker statistics to Elastic
-        :return: True if successful, False otherwise
+        :return: The amount of logged documents
         """
         es_index = (
             f"{self._stats.worker_stats_prefix_for_company(company_id)}"
@@ -532,8 +565,7 @@ class WorkerBLL:
                 _index=es_index,
                 _source=dict(
                     timestamp=timestamp,
-                    worker=worker,
-                    company=company_name,
+                    worker=worker_id,
                     task=task,
                     category=category,
                     metric=metric,
@@ -558,7 +590,7 @@ class WorkerBLL:
 
         es_res = elasticsearch.helpers.bulk(self.es_client, actions)
         added, errors = es_res[:2]
-        return (added == len(actions)) and not errors
+        return added
 
 
 @attr.s(auto_attribs=True)

apiserver/bll/workers/stats.py

@@ -13,6 +13,8 @@ log = config.logger(__file__)
 
 
 class WorkerStats:
+    min_chart_interval = config.get("services.workers.min_chart_interval_sec", 40)
+
     def __init__(self, es):
         self.es = es
 
@@ -71,9 +73,13 @@ class WorkerStats:
         Buckets with no metrics are not returned
         Note: all the statistics are retrieved as one ES query
         """
-        if request.from_date >= request.to_date:
+        from_date = request.from_date
+        to_date = request.to_date
+        if from_date >= to_date:
             raise bad_request.FieldsValueError("from_date must be less than to_date")
 
+        interval = max(request.interval, self.min_chart_interval)
+
         def get_dates_agg() -> dict:
             es_to_agg_types = (
                 ("avg", AggregationType.avg.value),
@@ -85,8 +91,11 @@ class WorkerStats:
                 "dates": {
                     "date_histogram": {
                         "field": "timestamp",
-                        "fixed_interval": f"{request.interval}s",
-                        "min_doc_count": 1,
+                        "fixed_interval": f"{interval}s",
+                        "extended_bounds": {
+                          "min": int(from_date) * 1000,
+                          "max": int(to_date) * 1000,
+                        }
                     },
                     "aggs": {
                         agg_type: {es_agg: {"field": "value"}}
@@ -118,7 +127,7 @@ class WorkerStats:
         }
 
         query_terms = [
-            QueryBuilder.dates_range(request.from_date, request.to_date),
+            QueryBuilder.dates_range(from_date, to_date),
             QueryBuilder.terms("metric", {item.key for item in request.items}),
         ]
         if request.worker_ids:
@@ -128,16 +137,16 @@ class WorkerStats:
         with translate_errors_context():
             data = self._search_company_stats(company_id, es_req)
 
-        return self._extract_results(data, request.items, request.split_by_variant)
+        cutoff_date = (to_date - 0.9 * interval) * 1000  # do not return the point for the incomplete last interval
+        return self._extract_results(data, request.items, request.split_by_variant, cutoff_date)
 
     @staticmethod
     def _extract_results(
-        data: dict, request_items: Sequence[StatItem], split_by_variant: bool
+        data: dict, request_items: Sequence[StatItem], split_by_variant: bool, cutoff_date
     ) -> dict:
         """
         Clean results returned from elastic search (remove "aggregations", "buckets" etc.),
         leave only aggregation types requested by the user and return a clean dictionary
-        and return a "clean" dictionary of
         :param data: aggregation data retrieved from ES
         :param request_items: aggs types requested by the user
         :param split_by_variant: if False then aggregate by metric type, otherwise metric type + variant
@@ -155,7 +164,7 @@ class WorkerStats:
             return {
                 "date": date["key"],
                 "count": date["doc_count"],
-                **{agg: date[agg]["value"] for agg in aggs_per_metric[metric_key]},
+                **{agg: date[agg]["value"] or 0.0 for agg in aggs_per_metric[metric_key]},
             }
 
         def extract_metric_results(
@@ -164,7 +173,7 @@ class WorkerStats:
             return [
                 extract_date_stats(date, metric_key)
                 for date in metric_or_variant["dates"]["buckets"]
-                if date["doc_count"]
+                if date["key"] <= cutoff_date
             ]
 
         def extract_variant_results(metric: dict) -> dict:
@@ -203,6 +212,7 @@ class WorkerStats:
         """
         if from_date >= to_date:
             raise bad_request.FieldsValueError("from_date must be less than to_date")
+        interval = max(interval, self.min_chart_interval)
 
         must = [QueryBuilder.dates_range(from_date, to_date)]
         if active_only:
@@ -215,6 +225,10 @@ class WorkerStats:
                     "date_histogram": {
                         "field": "timestamp",
                         "fixed_interval": f"{interval}s",
+                        "extended_bounds": {
+                          "min": int(from_date) * 1000,
+                          "max": int(to_date) * 1000,
+                        }
                     },
                     "aggs": {"workers_count": {"cardinality": {"field": "worker"}}},
                 }

apiserver/config/basic.py

@@ -6,7 +6,7 @@ from functools import reduce
 from os import getenv
 from os.path import expandvars
 from pathlib import Path
-from typing import List, Any, TypeVar, Sequence
+from typing import List, Any, TypeVar, Sequence, Set
 
 from boltons.iterutils import first
 from pyhocon import ConfigTree, ConfigFactory, ConfigValues
@@ -35,6 +35,7 @@ class BasicConfig:
         folder: str = None,
         verbose: bool = True,
         prefix: Sequence[str] = DEFAULT_PREFIXES,
+        exclude_files_from_base_folder: Sequence[str] = None,
     ):
         folder = (
             Path(folder)
@@ -44,6 +45,11 @@ class BasicConfig:
         if not folder.is_dir():
             raise ValueError("Invalid configuration folder")
 
+        self.exclude_files_from_base_folder = (
+            set(exclude_files_from_base_folder)
+            if exclude_files_from_base_folder
+            else set()
+        )
         self.verbose = verbose
 
         self.extra_config_path_override_var = [
@@ -85,7 +91,7 @@ class BasicConfig:
         return logging.getLogger(path)
 
     def _read_extra_env_config_values(self) -> ConfigTree:
-        """ Loads extra configuration from environment-injected values """
+        """Loads extra configuration from environment-injected values"""
         result = ConfigTree()
 
         for prefix in self.extra_config_values_env_key_prefix:
@@ -125,12 +131,18 @@ class BasicConfig:
     def _reload(self) -> ConfigTree:
         extra_config_values = self._read_extra_env_config_values()
 
-        configs = [self._read_recursive(path) for path in self._paths]
+        configs = [
+            self._read_recursive(
+                path,
+                exclude_files=(
+                    self.exclude_files_from_base_folder if idx == 0 else None
+                ),
+            )
+            for idx, path in enumerate(self._paths)
+        ]
 
         return reduce(
-            lambda last, config: self._merge_configs(
-                last, config, copy_trees=True
-            ),
+            lambda last, config: self._merge_configs(last, config, copy_trees=True),
             configs + [extra_config_values],
             ConfigTree(),
         )
@@ -141,9 +153,14 @@ class BasicConfig:
         for key, value in b.items():
             override = key.startswith(override_prefix)
             if override:
-                key = key[len(override_prefix):]
+                key = key[len(override_prefix) :]
             # if key is in both a and b and both values are dictionary then merge it otherwise override it
-            if not override and key in a and isinstance(a[key], ConfigTree) and isinstance(b[key], ConfigTree):
+            if (
+                not override
+                and key in a
+                and isinstance(a[key], ConfigTree)
+                and isinstance(b[key], ConfigTree)
+            ):
                 if copy_trees:
                     a[key] = a[key].copy()
                 cls._merge_configs(a[key], b[key], copy_trees=copy_trees)
@@ -156,13 +173,15 @@ class BasicConfig:
                 a[key] = value
                 if a.root:
                     if b.root:
-                        a.history[key] = a.history.get(key, []) + b.history.get(key, [value])
+                        a.history[key] = a.history.get(key, []) + b.history.get(
+                            key, [value]
+                        )
                     else:
                         a.history[key] = a.history.get(key, []) + [value]
 
         return a
 
-    def _read_recursive(self, conf_root) -> ConfigTree:
+    def _read_recursive(self, conf_root, exclude_files: Set[str]) -> ConfigTree:
         conf = ConfigTree()
 
         if not conf_root:
@@ -180,6 +199,8 @@ class BasicConfig:
             print(f"Loading config from {conf_root}")
 
         for file in conf_root.rglob("*.conf"):
+            if exclude_files and file.name in exclude_files:
+                continue
             key = ".".join(file.relative_to(conf_root).with_suffix("").parts)
             conf.put(key, self._read_single_file(file))
 

apiserver/config/default/apiserver.conf

@@ -58,6 +58,9 @@
         # verify user tokens
         verify_user_tokens: false
 
+        # If set then users that were created from secure credentials or fixed user settings and are no longer in these settings will be deleted on startup
+        delete_missing_autocreated_users: true
+
         # max token expiration timeout in seconds (1 year)
         max_expiration_sec: 31536000
 
@@ -72,6 +75,7 @@
             httponly: true  # allow only http to access the cookies (no JS etc)
             secure: false   # not using HTTPS
             domain: null    # Limit to localhost is not supported
+            samesite: Lax
             max_age: 99999999999
         }
 

apiserver/config/default/hosts.conf

@@ -2,10 +2,9 @@ fileserver = "http://localhost:8081"
 
 elastic {
   events {
-    hosts: [{host: "127.0.0.1", port: 9200}]
+    hosts: [{host: "127.0.0.1", port: 9200, scheme: http}]
     args {
       timeout: 60
-      dead_timeout: 10
       max_retries: 3
       retry_on_timeout: true
     }
@@ -13,10 +12,9 @@ elastic {
   }
 
   workers {
-    hosts: [{host:"127.0.0.1", port:9200}]
+    hosts: [{host:"127.0.0.1", port:9200, scheme: http}]
     args {
       timeout: 60
-      dead_timeout: 10
       max_retries: 3
       retry_on_timeout: true
     }

apiserver/config/default/secure.conf

@@ -1,13 +1,13 @@
 {
     http {
         session_secret {
-            apiserver: "Gx*gB-L2U8!Naqzd#8=7A4&+=In4H(da424H33ZTDQRGF6=FWw"
+            apiserver: "V8gcW3EneNDcNfO7G_TSUsWe7uLozyacc9_I33o7bxUo8rCN31VLRg"
         }
     }
 
     auth {
         # token sign secret
-        token_secret: "7E1ua3xP9GT2(cIQOfhjp+gwN6spBeCAmN-XuugYle00I=Wc+u"
+        token_secret: "Rq8FW84sSqVgq7WvBB_4EzNl9y8z8IGiDXX3C345_a5AZfcwZcwCIA"
     }
 
     credentials {
@@ -15,24 +15,29 @@
         apiserver {
             role: "system"
             user_key: "62T8CP7HGBC6647XF9314C2VY67RJO"
-            user_secret: "FhS8VZv_I4%6Mo$8S1BWc$n$=o1dMYSivuiWU-Vguq7qGOKskG-d+b@tn_Iq"
+            user_secret: "gaOfhDX2-bpkeI7-cwEcaMuGijxaG2UG3jbIvg4DxmVGF0LNI7rgvCb1-ne38IlBo1w"
+        }
+        fileserver {
+            role: "system"
+            user_key: "GSQWPEKSKNKF354LC9V6BHXKTYFD5I"
+            user_secret: "tuBXcGQBECsEhcNiK2kiWi750z9r8Z85XrQ9V0c24huTuCb2xf2X1nKG"
         }
         webserver {
             role: "system"
             user_key: "EYVQ385RW7Y2QQUH88CZ7DWIQ1WUHP"
-            user_secret: "yfc8KQo*GMXb*9p((qcYC7ByFIpF7I&4VH3BfUYXH%o9vX1ZUZQEEw1Inc)S"
+            user_secret: "XhkH6a6ds9JBnM_MrahYyYdO-wS2bqFSm8gl-V0UZXH26Ydd6Eyi28TeBEoSr6Z3Bes"
             revoke_in_fixed_mode: true
         }
         services_agent {
             role: "admin"
-            user_key: "P4BMJA7RK3TKBXGSY8OAA1FA8TOD11"
-            user_secret: "9LsgSfa0SYz0zli1_c500ZcLqanre2xkWOpepyt1w-BKK3_DKPHrtoj3JSHvyy8bIi0"
+            user_key: ""
+            user_secret: ""
         }
         tests {
             role: "user"
             display_name: "Default User"
             user_key: "EGRTCO8JMSIGI6S39GTP43NFWXDQOW"
-            user_secret: "x!XTov_G-#vspE*Y(h$Anm&DIc5Ou-F)jsl$PdOyj5wG1&E!Z8"
+            user_secret: "LPEJbGJ6bK4tujQcmrD3i1dbMBDdwUwelVa-LG0K0FFmY9bzH_H0Sw"
             revoke_in_fixed_mode: true
         }
     }

apiserver/config/default/services/async_urls_delete.conf

@@ -9,4 +9,5 @@ fileserver {
     # Can be in the form <schema>://host:port/path or /path
     url_prefixes: ["https://files.community-master.hosted.allegro.ai/"]
     timeout_sec: 300
+    token_expiration_sec: 600
 }

apiserver/config/default/services/serving.conf

@@ -0,0 +1,7 @@
+default_container_timeout_sec: 600
+# Auto-register unknown serving containers on status reports and other calls
+container_auto_register: true
+# Assume unknow serving containers have unregistered (i.e. do not raise unregistered error)
+container_auto_unregister: true
+# The minimal sampling interval for serving model monitor chars
+min_chart_interval_sec: 40

apiserver/config/default/services/storage_credentials.conf

@@ -18,8 +18,9 @@ aws {
             {
                 # This will apply to all buckets in this host (unless key/value is specifically provided for a given bucket)
                 host: "localhost:9000"
-                key: "evg_user"
-                secret: "evg_pass"
+                key: "minioadmin"
+                secret: "minioadmin"
+                # region: my-server
                 multipart: false
                 secure: false
             }

apiserver/config/default/services/tasks.conf

@@ -23,4 +23,6 @@ hyperparam_values {
 max_last_metrics: 2000
 
 # if set then call to tasks.delete/cleanup does not wait for ES events deletion
-async_events_delete: false
+async_events_delete: true
+# do not use async_delete if the deleted task has amount of events lower than this threshold
+async_events_delete_threshold: 100000

apiserver/config/default/services/workers.conf

@@ -0,0 +1,5 @@
+default_worker_timeout_sec: 600
+default_cluster_timeout_sec: 600
+
+# The minimal sampling interval for resource dashboard and worker activity charts
+min_chart_interval_sec: 40

apiserver/database/__init__.py

@@ -81,7 +81,7 @@ class DatabaseFactory:
             entry = cls._create_db_entry(alias=alias, settings=db_entries.get(key))
 
             if override_connection_string:
-                con_str = f"{override_connection_string.rstrip('/')}/{key}"
+                con_str = furl(override_connection_string).add(path=key).url
                 log.info(f"Using override mongodb connection string for {alias}: {con_str}")
                 entry.host = con_str
             else:

apiserver/database/errors.py

@@ -5,7 +5,7 @@ from textwrap import shorten
 
 import dpath
 from dpath.exceptions import InvalidKeyName
-from elasticsearch import ElasticsearchException
+from elastic_transport import TransportError, ApiError
 from elasticsearch.helpers import BulkIndexError
 from jsonmodels.errors import ValidationError as JsonschemaValidationError
 from mongoengine.errors import (
@@ -210,9 +210,9 @@ def translate_errors_context(message=None, **kwargs):
         raise errors.bad_request.ValidationError(e.args[0])
     except BulkIndexError as e:
         ElasticErrorsHandler.bulk_error(e, message, **kwargs)
-    except ElasticsearchException as e:
+    except (TransportError, ApiError) as e:
         raise errors.server_error.DataError(e, message, **kwargs)
     except InvalidKeyName:
         raise errors.server_error.DataError("invalid empty key encountered in data")
-    except Exception as ex:
+    except Exception:
         raise

apiserver/database/model/auth.py

@@ -4,6 +4,7 @@ from mongoengine import (
     EmbeddedDocumentListField,
     EmailField,
     DateTimeField,
+    BooleanField,
 )
 
 from apiserver.database import Database, strict
@@ -76,3 +77,6 @@ class User(DbModelMixin, AuthDocument):
 
     email = EmailField(unique=True, sparse=True)
     """ Email uniquely identifying the user """
+
+    autocreated = BooleanField(default=False)
+    """ Set to true if the user was auto created based on config settings"""

apiserver/database/model/base.py

@@ -1,5 +1,6 @@
 import re
-from collections import namedtuple, defaultdict
+from collections import defaultdict
+from datetime import datetime
 from functools import reduce, partial
 from typing import (
     Collection,
@@ -106,7 +107,18 @@ class GetMixin(PropsMixin):
         ("_any_", "_or_"): lambda a, b: a | b,
         ("_all_", "_and_"): lambda a, b: a & b,
     }
-    MultiFieldParameters = namedtuple("MultiFieldParameters", "pattern fields")
+
+    @attr.s(auto_attribs=True)
+    class MultiFieldParameters:
+        fields: Sequence[str]
+        pattern: str = None
+        datetime: Union[list, str] = None
+
+        def __attrs_post_init__(self):
+            if not any(f is not None for f in (self.pattern, self.datetime)):
+                raise ValueError("Either 'pattern' or 'datetime' should be provided")
+            if all(f is not None for f in (self.pattern, self.datetime)):
+                raise ValueError("Only one of the 'pattern' and 'datetime' can be provided")
 
     _numeric_locale = {"locale": "en_US", "numericOrdering": True}
     _field_collation_overrides = {}
@@ -145,9 +157,10 @@ class GetMixin(PropsMixin):
             "__$any": Q.OR,
             "__$or": Q.OR,
         }
-        default_operator = Q.OR
+        default_global_operator = Q.AND
+        default_context = Q.OR
+        # not_all modifier currently not supported due to the backwards compatibility
         mongo_modifiers = {
-            # not_all modifier currently not supported due to the backwards compatibility
             Q.AND: {True: "all", False: "nin"},
             Q.OR: {True: "in", False: "nin"},
         }
@@ -164,24 +177,22 @@ class GetMixin(PropsMixin):
             self.allow_empty = False
             self.global_operator = None
             self.actions = defaultdict(list)
-            self.explicit_operator = False
 
             self._support_legacy = legacy
-            current_context = self.default_operator
+            current_context = self.default_context
             for d in self._get_next_term(data):
                 if d.operator is not None:
                     current_context = d.operator
                     self._support_legacy = False
                     if self.global_operator is None:
                         self.global_operator = d.operator
-                        self.explicit_operator = True
                     continue
 
                 if self.global_operator is None:
-                    self.global_operator = self.default_operator
+                    self.global_operator = self.default_global_operator
 
                 if d.reset:
-                    current_context = self.default_operator
+                    current_context = self.default_context
                     self._support_legacy = legacy
                     continue
 
@@ -194,11 +205,9 @@ class GetMixin(PropsMixin):
                 )
 
             if self.global_operator is None:
-                self.global_operator = self.default_operator
+                self.global_operator = self.default_global_operator
 
-        def _get_next_term(
-            self, data: Sequence[str]
-        ) -> Generator[Term, None, None]:
+        def _get_next_term(self, data: Sequence[str]) -> Generator[Term, None, None]:
             unary_operator = None
             for value in data:
                 if value is None:
@@ -232,12 +241,18 @@ class GetMixin(PropsMixin):
                     operator = self._operators.get(value)
                     if operator is None:
                         raise FieldsValueError(
-                            "Unsupported operator", field=self._field, operator=value,
+                            "Unsupported operator",
+                            field=self._field,
+                            operator=value,
                         )
                     yield self.Term(operator=operator)
                     continue
 
-                if not unary_operator and self._support_legacy and value.startswith("-"):
+                if (
+                    not unary_operator
+                    and self._support_legacy
+                    and value.startswith("-")
+                ):
                     value = value[1:]
                     if not value:
                         raise FieldsValueError(
@@ -319,6 +334,8 @@ class GetMixin(PropsMixin):
                 specific rules on handling values). Only items matching ALL of these conditions will be retrieved.
             - <any|all>: {fields: [<field1>, <field2>, ...], pattern: <pattern>} Will query for items where any or all
                 provided fields match the provided pattern.
+            - <any|all>: {fields: [<field1>, <field2>, ...], datetime: <datetime condition>} Will query for items where any or all
+                provided datetime fields match the provided condition.
         :return: mongoengine.Q query object
         """
         return cls._prepare_query_no_company(
@@ -372,6 +389,46 @@ class GetMixin(PropsMixin):
             return cls._try_convert_to_numeric(value)
         return value
 
+    @classmethod
+    def _get_dates_query(cls, field: str, data: Union[list, str]) -> Union[Q, dict]:
+        """
+        Return dates query for the field
+        If the data is 2 values array and none of the values starts from dates comparison operations
+        then return the simplified range query
+        Otherwise return the dictionary of dates conditions
+        """
+        if not isinstance(data, list):
+            data = [data]
+
+        if len(data) == 2 and not any(
+                d.startswith(mod)
+                for d in data
+                if d is not None
+                for mod in ACCESS_MODIFIER
+        ):
+            return cls.get_range_field_query(field, data)
+
+        dict_query = {}
+        for d in data:
+            m = ACCESS_REGEX.match(d)
+            if not m:
+                continue
+
+            try:
+                value = parse_datetime(m.group("value"))
+                prefix = m.group("prefix")
+                modifier = ACCESS_MODIFIER.get(prefix)
+                f = (
+                    field
+                    if not modifier
+                    else "__".join((field, modifier))
+                )
+                dict_query[f] = value
+            except (ValueError, OverflowError):
+                pass
+
+        return dict_query
+
     @classmethod
     def _prepare_query_no_company(
         cls, parameters=None, parameters_options=QueryParameterOptions()
@@ -402,12 +459,25 @@ class GetMixin(PropsMixin):
                 parameters = {
                     k: cls._get_fixed_field_value(k, v) for k, v in parameters.items()
                 }
+                filters = parameters.pop("filters", {})
+                if not isinstance(filters, dict):
+                    raise FieldsValueError(
+                        "invalid value type, string expected",
+                        field=filters,
+                        value=str(filters),
+                    )
                 opts = parameters_options
                 for field in opts.pattern_fields:
                     pattern = parameters.pop(field, None)
                     if pattern:
                         dict_query[field] = RegexWrapper(pattern)
 
+                for field, data in cls._pop_matching_params(
+                    patterns=opts.list_fields, parameters=filters
+                ).items():
+                    query &= cls.get_list_filter_query(field, data)
+                    parameters.pop(field, None)
+
                 for field, data in cls._pop_matching_params(
                     patterns=opts.list_fields, parameters=parameters
                 ).items():
@@ -429,33 +499,11 @@ class GetMixin(PropsMixin):
                 for field in opts.datetime_fields or []:
                     data = parameters.pop(field, None)
                     if data is not None:
-                        if not isinstance(data, list):
-                            data = [data]
-                        # date time fields also support simplified range queries. Check if this is the case
-                        if len(data) == 2 and not any(
-                            d.startswith(mod)
-                            for d in data
-                            if d is not None
-                            for mod in ACCESS_MODIFIER
-                        ):
-                            query &= cls.get_range_field_query(field, data)
-                        else:
-                            for d in data:  # type: str
-                                m = ACCESS_REGEX.match(d)
-                                if not m:
-                                    continue
-                                try:
-                                    value = parse_datetime(m.group("value"))
-                                    prefix = m.group("prefix")
-                                    modifier = ACCESS_MODIFIER.get(prefix)
-                                    f = (
-                                        field
-                                        if not modifier
-                                        else "__".join((field, modifier))
-                                    )
-                                    dict_query[f] = value
-                                except (ValueError, OverflowError):
-                                    pass
+                        dates_q = cls._get_dates_query(field, data)
+                        if isinstance(dates_q, Q):
+                            query &= dates_q
+                        elif isinstance(dates_q, dict):
+                            dict_query.update(dates_q)
 
                 for field, value in parameters.items():
                     for keys, func in cls._multi_field_param_prefix.items():
@@ -467,27 +515,40 @@ class GetMixin(PropsMixin):
                             raise MakeGetAllQueryError("incorrect field format", field)
                         if not data.fields:
                             break
-                        if any("._" in f for f in data.fields):
-                            q = reduce(
-                                lambda a, x: func(
-                                    a,
-                                    RegexQ(
-                                        __raw__={
-                                            x: {"$regex": data.pattern, "$options": "i"}
-                                        }
+                        if data.pattern is not None:
+                            if any("._" in f for f in data.fields):
+                                q = reduce(
+                                    lambda a, x: func(
+                                        a,
+                                        RegexQ(
+                                            __raw__={
+                                                x: {"$regex": data.pattern, "$options": "i"}
+                                            }
+                                        ),
                                     ),
-                                ),
-                                data.fields,
-                                RegexQ(),
-                            )
+                                    data.fields,
+                                    RegexQ(),
+                                )
+                            else:
+                                regex = RegexWrapper(data.pattern, flags=re.IGNORECASE)
+                                sep_fields = [f.replace(".", "__") for f in data.fields]
+                                q = reduce(
+                                    lambda a, x: func(a, RegexQ(**{x: regex})),
+                                    sep_fields,
+                                    RegexQ(),
+                                )
                         else:
-                            regex = RegexWrapper(data.pattern, flags=re.IGNORECASE)
-                            sep_fields = [f.replace(".", "__") for f in data.fields]
-                            q = reduce(
-                                lambda a, x: func(a, RegexQ(**{x: regex})),
-                                sep_fields,
-                                RegexQ(),
-                            )
+                            date_fields = [field for field in data.fields if field in opts.datetime_fields]
+                            if not date_fields:
+                                break
+
+                            q = Q()
+                            for date_f in date_fields:
+                                dates_q = cls._get_dates_query(date_f, data.datetime)
+                                if isinstance(dates_q, dict):
+                                    dates_q = RegexQ(**dates_q)
+                                q = func(q, dates_q)
+
                         query = query & q
         except APIError:
             raise
@@ -531,6 +592,149 @@ class GetMixin(PropsMixin):
 
         return q
 
+    @attr.s(auto_attribs=True)
+    class ListQueryFilter:
+        """
+        Deserialize filters data and build db_query object that represents it with the corresponding
+        mongo engine operations
+        Each part has include and exclude lists that map to mongoengine operations as following:
+        "any"
+          - include -> 'in'
+          - exclude -> 'not_all'
+          - combined by 'or' operation
+        "all"
+          - include -> 'all'
+          - exclude -> 'nin'
+          - combined by 'and' operation
+        "op" optional parameter for combining "and" and "all" parts. Can be "and" or "or". The default is "and"
+        """
+
+        _and_op = "and"
+        _or_op = "or"
+        _allowed_op = [_and_op, _or_op]
+        _db_modifiers: Mapping = {
+            (Q.OR, True): "in",
+            (Q.OR, False): "not__all",
+            (Q.AND, True): "all",
+            (Q.AND, False): "nin",
+        }
+
+        @attr.s(auto_attribs=True)
+        class ListFilter:
+            include: Sequence[str] = []
+            exclude: Sequence[str] = []
+
+            @classmethod
+            def from_dict(cls, d: Mapping):
+                if d is None:
+                    return None
+                return cls(**d)
+
+        any: ListFilter = attr.ib(converter=ListFilter.from_dict, default=None)
+        all: ListFilter = attr.ib(converter=ListFilter.from_dict, default=None)
+        op: str = attr.ib(default="and")
+        db_query: dict = attr.ib(init=False)
+
+        # noinspection PyUnresolvedReferences
+        @op.validator
+        def op_validator(self, _, value):
+            if value not in self._allowed_op:
+                raise ValueError(
+                    f"Invalid list query filter operator: {value}. "
+                    f"Should be one of {str(self._allowed_op)}"
+                )
+
+        @property
+        def and_op(self) -> bool:
+            return self.op == self._and_op
+
+        def __attrs_post_init__(self):
+            self.db_query = {}
+            for op, conditions in ((Q.OR, self.any), (Q.AND, self.all)):
+                if not conditions:
+                    continue
+
+                operations = {}
+                for vals, include in (
+                    (conditions.include, True),
+                    (conditions.exclude, False),
+                ):
+                    if not vals:
+                        continue
+
+                    unique = set(vals)
+                    if None in unique:
+                        # noinspection PyTypeChecker
+                        unique.remove(None)
+                        if include:
+                            operations["size"] = 0
+                        else:
+                            operations["not__size"] = 0
+
+                    if not unique:
+                        continue
+
+                    operations[self._db_modifiers[(op, include)]] = list(unique)
+
+                self.db_query[op] = operations
+
+        @classmethod
+        def from_data(cls, field, data: Mapping):
+            if not isinstance(data, dict):
+                raise errors.bad_request.ValidationError(
+                    "invalid filter for field, dictionary expected",
+                    field=field,
+                    value=str(data),
+                )
+
+            try:
+                return cls(**data)
+            except Exception as ex:
+                raise errors.bad_request.ValidationError(
+                    field=field,
+                    value=str(ex),
+                )
+
+    @classmethod
+    def get_list_filter_query(
+        cls, field: str, data: Mapping
+    ) -> Union[RegexQ, RegexQCombination]:
+        if not data:
+            return RegexQ()
+
+        filter_ = cls.ListQueryFilter.from_data(field, data)
+
+        mongoengine_field = field.replace(".", "__")
+        queries = []
+        for op, actions in filter_.db_query.items():
+            if not actions:
+                continue
+
+            ops = []
+            for action, vals in actions.items():
+                # cannot just check vals here since 0 is acceptable value
+                if vals is None or vals == []:
+                    continue
+
+                ops.append(RegexQ(**{f"{mongoengine_field}__{action}": vals}))
+
+            if not ops:
+                continue
+
+            if len(ops) == 1:
+                queries.extend(ops)
+                continue
+
+            queries.append(RegexQCombination(operation=op, children=ops))
+
+        if not queries:
+            return RegexQ()
+        if len(queries) == 1:
+            return queries[0]
+
+        operation = Q.AND if filter_.and_op else Q.OR
+        return RegexQCombination(operation=operation, children=queries)
+
     @classmethod
     def get_list_field_query(cls, field: str, data: Sequence[Optional[str]]) -> RegexQ:
         """
@@ -639,7 +843,7 @@ class GetMixin(PropsMixin):
 
     @classmethod
     def get_projection(cls, parameters, override_projection=None, **__):
-        """ Extract a projection list from the provided dictionary. Supports an override projection. """
+        """Extract a projection list from the provided dictionary. Supports an override projection."""
         if override_projection is not None:
             return override_projection
         if not parameters:
@@ -653,7 +857,8 @@ class GetMixin(PropsMixin):
         """Return include and exclude lists based on passed projection and class definition"""
         if projection:
             include, exclude = partition(
-                projection, key=lambda x: x[0] != ProjectionHelper.exclusion_prefix,
+                projection,
+                key=lambda x: x[0] != ProjectionHelper.exclusion_prefix,
             )
         else:
             include, exclude = [], []
@@ -900,7 +1105,9 @@ class GetMixin(PropsMixin):
                 projection_fields=projection_fields,
             )
             return cls.get_data_with_scroll_support(
-                query_dict=query_dict, data_getter=data_getter, ret_params=ret_params,
+                query_dict=query_dict,
+                data_getter=data_getter,
+                ret_params=ret_params,
             )
 
         return cls._get_many_no_company(
@@ -913,7 +1120,9 @@ class GetMixin(PropsMixin):
 
     @classmethod
     def get_many_public(
-        cls, query: Q = None, projection: Collection[str] = None,
+        cls,
+        query: Q = None,
+        projection: Collection[str] = None,
     ):
         """
         Fetch all public documents matching a provided query.
@@ -1131,22 +1340,6 @@ class GetMixin(PropsMixin):
             )
         return result
 
-    @classmethod
-    def get_many_for_writing(cls, company, *args, **kwargs):
-        result = cls.get_many(
-            company=company,
-            *args,
-            **dict(return_dicts=False, **kwargs),
-            allow_public=True,
-        )
-        forbidden_objects = {obj.id for obj in result if not obj.company}
-        if forbidden_objects:
-            object_name = cls.__name__.lower()
-            raise errors.forbidden.NoWritePermission(
-                f"cannot modify public {object_name}(s), ids={tuple(forbidden_objects)}"
-            )
-        return result
-
 
 class UpdateMixin(object):
     __user_set_allowed_fields = None
@@ -1206,7 +1399,7 @@ class UpdateMixin(object):
 
 
 class DbModelMixin(GetMixin, ProperDictMixin, UpdateMixin):
-    """ Provide convenience methods for a subclass of mongoengine.Document """
+    """Provide convenience methods for a subclass of mongoengine.Document"""
 
     @classmethod
     def aggregate(
@@ -1234,25 +1427,31 @@ class DbModelMixin(GetMixin, ProperDictMixin, UpdateMixin):
     def set_public(
         cls: Type[Document],
         company_id: str,
+        user_id: str,
         ids: Sequence[str],
         invalid_cls: Type[BaseError],
         enabled: bool = True,
     ):
         if enabled:
             items = list(cls.objects(id__in=ids, company=company_id).only("id"))
-            update = dict(set__company_origin=company_id, set__company="")
+            update: dict = dict(set__company_origin=company_id, set__company="")
         else:
             items = list(
                 cls.objects(
-                    id__in=ids, company__in=(None, ""), company_origin=company_id
+                    id__in=ids, company="", company_origin=company_id
                 ).only("id")
             )
-            update = dict(set__company=company_id, unset__company_origin=1)
+            update: dict = dict(set__company=company_id, unset__company_origin=1)
 
         if len(items) < len(ids):
             missing = tuple(set(ids).difference(i.id for i in items))
             raise invalid_cls(ids=missing)
 
+        if hasattr(cls, "last_change"):
+            update["set__last_change"] = datetime.utcnow()
+        if hasattr(cls, "last_changed_by"):
+            update["set__last_changed_by"] = user_id
+
         return {"updated": cls.objects(id__in=ids).update(**update)}
 
 

apiserver/database/model/model.py

@@ -37,10 +37,18 @@ class Model(AttributedDocument):
             "project",
             "task",
             "last_update",
-            ("company", "framework"),
+            ("company", "last_update"),
             ("company", "name"),
-            ("company", "user"),
             ("company", "uri"),
+            # distinct queries support
+            ("company", "tags"),
+            ("company", "system_tags"),
+            ("company", "project", "tags"),
+            ("company", "project", "system_tags"),
+            ("company", "user"),
+            ("company", "project", "user"),
+            ("company", "framework"),
+            ("company", "project", "framework"),
             {
                 "name": "%s.model.main_text_index" % Database.backend,
                 "fields": ["$name", "$id", "$comment", "$parent", "$task", "$project"],
@@ -71,8 +79,8 @@ class Model(AttributedDocument):
             "parent",
             "metadata.*",
         ),
-        range_fields=("last_metrics.*", "last_iteration"),
-        datetime_fields=("last_update",),
+        range_fields=("created", "last_metrics.*", "last_iteration"),
+        datetime_fields=("last_update", "last_change"),
     )
 
     id = StringField(primary_key=True)
@@ -90,6 +98,8 @@ class Model(AttributedDocument):
     labels = ModelLabels()
     ready = BooleanField(required=True)
     last_update = DateTimeField()
+    last_change = DateTimeField()
+    last_changed_by = StringField()
     ui_cache = SafeDictField(
         default=dict, user_set_allowed=True, exclude_by_default=True
     )

apiserver/database/model/storage_settings.py

@@ -0,0 +1,76 @@
+from mongoengine import (
+    Document,
+    EmbeddedDocument,
+    StringField,
+    DateTimeField,
+    EmbeddedDocumentListField,
+    EmbeddedDocumentField,
+    BooleanField,
+)
+
+from apiserver.database import Database, strict
+from apiserver.database.model import DbModelMixin
+from apiserver.database.model.base import ProperDictMixin
+
+class AWSBucketSettings(EmbeddedDocument, ProperDictMixin):
+    bucket = StringField()
+    subdir = StringField()
+    host = StringField()
+    key = StringField()
+    secret = StringField()
+    token = StringField()
+    multipart = BooleanField()
+    acl = StringField()
+    secure = BooleanField()
+    region = StringField()
+    verify = BooleanField()
+    use_credentials_chain = BooleanField()
+
+
+class AWSSettings(EmbeddedDocument, DbModelMixin):
+    key = StringField()
+    secret = StringField()
+    region = StringField()
+    token = StringField()
+    use_credentials_chain = BooleanField()
+    buckets = EmbeddedDocumentListField(AWSBucketSettings)
+
+
+class GoogleBucketSettings(EmbeddedDocument, ProperDictMixin):
+    bucket = StringField()
+    subdir = StringField()
+    project = StringField()
+    credentials_json = StringField()
+
+
+class GoogleStorageSettings(EmbeddedDocument, DbModelMixin):
+    project = StringField()
+    credentials_json = StringField()
+    buckets = EmbeddedDocumentListField(GoogleBucketSettings)
+
+
+class AzureStorageContainerSettings(EmbeddedDocument, ProperDictMixin):
+    account_name = StringField(required=True)
+    account_key = StringField(required=True)
+    container_name = StringField()
+
+
+class AzureStorageSettings(EmbeddedDocument, DbModelMixin):
+    containers = EmbeddedDocumentListField(AzureStorageContainerSettings)
+
+
+class StorageSettings(DbModelMixin, Document):
+    meta = {
+        "db_alias": Database.backend,
+        "strict": strict,
+        "indexes": [
+            "company"
+        ],
+    }
+
+    id = StringField(primary_key=True)
+    company = StringField(required=True, unique=True)
+    last_update = DateTimeField()
+    aws: AWSSettings = EmbeddedDocumentField(AWSSettings)
+    google: GoogleStorageSettings = EmbeddedDocumentField(GoogleStorageSettings)
+    azure: AzureStorageSettings = EmbeddedDocumentField(AzureStorageSettings)

apiserver/database/model/task/metrics.py

@@ -5,6 +5,7 @@ from mongoengine import (
     LongField,
     EmbeddedDocumentField,
     IntField,
+    FloatField,
 )
 
 from apiserver.database.fields import SafeMapField
@@ -23,6 +24,10 @@ class MetricEvent(EmbeddedDocument):
     min_value_iteration = IntField()
     max_value = DynamicField()  # for backwards compatibility reasons
     max_value_iteration = IntField()
+    first_value = FloatField()
+    first_value_iteration = IntField()
+    count = IntField()
+    mean_value = FloatField()
 
 
 class EventStats(EmbeddedDocument):

apiserver/database/model/task/task.py

@@ -183,9 +183,8 @@ class Task(AttributedDocument):
             "status_changed",
             "models.input.model",
             ("company", "name"),
-            ("company", "user"),
             ("company", "status", "type"),
-            ("company", "system_tags", "last_update"),
+            ("company", "last_update", "system_tags"),
             ("company", "type", "system_tags", "status"),
             ("company", "project", "type", "system_tags", "status"),
             ("status", "last_update"),  # for maintenance tasks
@@ -193,6 +192,17 @@ class Task(AttributedDocument):
                 "fields": ["company", "project"],
                 "collation": AttributedDocument._numeric_locale,
             },
+            # distinct queries support
+            ("company", "tags"),
+            ("company", "system_tags"),
+            ("company", "project", "tags"),
+            ("company", "project", "system_tags"),
+            ("company", "user"),
+            ("company", "project", "user"),
+            ("company", "parent"),
+            ("company", "project", "parent"),
+            ("company", "type"),
+            ("company", "project", "type"),
             {
                 "name": "%s.task.main_text_index" % Database.backend,
                 "fields": [
@@ -230,11 +240,13 @@ class Task(AttributedDocument):
             "project",
             "parent",
             "hyperparams.*",
+            "execution.queue",
+            "models.input.model",
         ),
-        range_fields=("started", "active_duration", "last_metrics.*", "last_iteration"),
-        datetime_fields=("status_changed", "last_update"),
+        range_fields=("created", "started", "active_duration", "last_metrics.*", "last_iteration"),
+        datetime_fields=("status_changed", "last_update", "last_change"),
         pattern_fields=("name", "comment", "report"),
-        fields=("execution.queue", "runtime.*", "models.input.model"),
+        fields=("runtime.*",),
     )
 
     id = StringField(primary_key=True)

apiserver/database/model/user.py

@@ -20,4 +20,5 @@ class User(DbModelMixin, Document):
     given_name = StringField(user_set_allowed=True)
     avatar = StringField()
     preferences = DynamicField(default="", exclude_by_default=True)
+    created_in_version = StringField()
     created = DateTimeField()

apiserver/database/utils.py

@@ -121,8 +121,8 @@ def init_cls_from_base(cls, instance):
     )
 
 
-def get_company_or_none_constraint(company=None):
-    return Q(company__in=(company, None, "")) | Q(company__exists=False)
+def get_company_or_none_constraint(company=""):
+    return Q(company__in=list({company, ""}))
 
 
 def field_does_not_exist(field: str, empty_value=None, is_list=False) -> Q:

apiserver/documentation/api_versions.md

@@ -0,0 +1,23 @@
+### Supported api versions
+
+| Release | ApiVersion |
+|---------|------------|
+| v1.17   | 2.31       |
+| v1.16   | 2.30       |
+| v1.15   | 2.29       |
+| v1.14   | 2.28       |
+| v1.13   | 2.27       |
+| v1.12   | 2.26       |
+| v1.11   | 2.25       |
+| v1.10   | 2.24       |
+| v1.9    | 2.23       |
+| v1.8    | 2.22       |
+| v1.7    | 2.21       |
+| v1.6    | 2.20       |
+| v1.5    | 2.19       |
+| v1.4    | 2.18       |
+| v1.3    | 2.17       |
+| v1.2    | 2.16       |
+| v1.1    | 2.15       |
+| v1.0    | 2.14       |
+| v0.17   | 2.13       |

apiserver/elastic/apply_mappings.py

@@ -4,34 +4,89 @@ Apply elasticsearch mappings to given hosts.
 """
 import argparse
 import json
+import logging
 from pathlib import Path
 from typing import Optional, Sequence, Tuple
 
-from elasticsearch import Elasticsearch
+from elasticsearch import Elasticsearch, exceptions
 
 HERE = Path(__file__).resolve().parent
+logging.getLogger("elasticsearch").setLevel(logging.WARNING)
+logging.getLogger("elastic_transport").setLevel(logging.WARNING)
 
 
 def apply_mappings_to_cluster(
-    hosts: Sequence, key: Optional[str] = None, es_args: dict = None, http_auth: Tuple = None
+    hosts: Sequence,
+    key: Optional[str] = None,
+    es_args: dict = None,
+    http_auth: Tuple = None,
 ):
     """Hosts maybe a sequence of strings or dicts in the form {"host": <host>, "port": <port>}"""
 
-    def _send_template(f):
-        with f.open() as json_data:
-            data = json.load(json_data)
-            template_name = f.stem
-            res = es.indices.put_template(name=template_name, body=data)
-            return {"mapping": template_name, "result": res}
+    def _send_component_template(ct_file):
+        with ct_file.open() as json_data:
+            body = json.load(json_data)
+            template_name = f"{ct_file.stem}"
+            res = es.cluster.put_component_template(name=template_name, body=body)
+            return {"component_template": template_name, "result": res}
 
-    p = HERE / "mappings"
-    if key:
-        files = (p / key).glob("*.json")
-    else:
-        files = p.glob("**/*.json")
+    def _send_index_template(it_file):
+        with it_file.open() as json_data:
+            body = json.load(json_data)
+            template_name = f"{it_file.stem}"
+            res = es.indices.put_index_template(name=template_name, body=body)
+            return {"index_template": template_name, "result": res}
+
+    # def _send_legacy_template(f):
+    #     with f.open() as json_data:
+    #         data = json.load(json_data)
+    #         template_name = f.stem
+    #         res = es.indices.put_template(name=template_name, body=data)
+    #         return {"mapping": template_name, "result": res}
+
+    def _delete_legacy_templates(legacy_folder):
+        res_list = []
+        for lt in legacy_folder.glob("*.json"):
+            template_name = lt.stem
+            try:
+                if not es.indices.get_template(name=template_name):
+                    continue
+                res = es.indices.delete_template(name=template_name)
+            except exceptions.NotFoundError:
+                continue
+            res_list.append({"deleted legacy mapping": template_name, "result": res})
+
+        return res_list
 
     es = Elasticsearch(hosts=hosts, http_auth=http_auth, **(es_args or {}))
-    return [_send_template(f) for f in files]
+    root = HERE / "index_templates"
+    if key:
+        folders = [root / key]
+    else:
+        folders = [f for f in root.iterdir() if f.is_dir()]
+
+    ret = []
+    for f in folders:
+        for ct in (f / "component_templates").glob("*.json"):
+            ret.append(_send_component_template(ct))
+        for it in f.glob("*.json"):
+            ret.append(_send_index_template(it))
+
+    legacy_root = HERE / "mappings"
+    for f in folders:
+        legacy_f = legacy_root / f.stem
+        if not legacy_f.exists() or not legacy_f.is_dir():
+            continue
+        ret.extend(_delete_legacy_templates(legacy_f))
+
+    return ret
+    # p = HERE / "mappings"
+    # if key:
+    #     files = (p / key).glob("*.json")
+    # else:
+    #     files = p.glob("**/*.json")
+    #
+    # return [_send_template(f) for f in files]
 
 
 def parse_args():

apiserver/elastic/index_templates/events/component_templates/events_common.json

@@ -0,0 +1,48 @@
+{
+  "template": {
+    "settings": {
+      "number_of_replicas": 0,
+      "number_of_shards": 1
+    },
+    "mappings": {
+      "_source": {
+        "enabled": true
+      },
+      "properties": {
+        "@timestamp": {
+          "type": "date"
+        },
+        "task": {
+          "type": "keyword"
+        },
+        "type": {
+          "type": "keyword"
+        },
+        "worker": {
+          "type": "keyword"
+        },
+        "timestamp": {
+          "type": "date"
+        },
+        "iter": {
+          "type": "long"
+        },
+        "metric": {
+          "type": "keyword"
+        },
+        "variant": {
+          "type": "keyword"
+        },
+        "value": {
+          "type": "float"
+        },
+        "company_id": {
+          "type": "keyword"
+        },
+        "model_event": {
+          "type": "boolean"
+        }
+      }
+    }
+  }
+}

apiserver/elastic/index_templates/events/events_log.json

@@ -0,0 +1,18 @@
+{
+  "index_patterns": "events-log-*",
+  "template": {
+    "mappings": {
+      "properties": {
+        "msg": {
+          "type": "text",
+          "index": false
+        },
+        "level": {
+          "type": "keyword"
+        }
+      }
+    }
+  },
+  "priority": 500,
+  "composed_of": ["events_common"]
+}

apiserver/elastic/index_templates/events/events_plot.json

@@ -0,0 +1,18 @@
+{
+  "index_patterns": "events-plot-*",
+  "template": {
+    "mappings": {
+      "properties": {
+        "plot_str": {
+          "type": "text",
+          "index": false
+        },
+        "plot_data": {
+          "type": "binary"
+        }
+      }
+    }
+  },
+  "priority": 500,
+  "composed_of": ["events_common"]
+}

apiserver/elastic/index_templates/events/events_training_debug_image.json

@@ -0,0 +1,17 @@
+{
+  "index_patterns": "events-training_debug_image-*",
+  "template": {
+    "mappings": {
+      "properties": {
+        "key": {
+          "type": "keyword"
+        },
+        "url": {
+          "type": "keyword"
+        }
+      }
+    }
+  },
+  "priority": 500,
+  "composed_of": ["events_common"]
+}

apiserver/elastic/index_templates/events/events_training_stats_scalar.json

@@ -0,0 +1,5 @@
+{
+  "index_patterns": "events-training_stats_scalar-*",
+  "priority": 500,
+  "composed_of": ["events_common"]
+}

apiserver/elastic/index_templates/workers/queue_metrics.json

@@ -0,0 +1,31 @@
+{
+  "index_patterns": "queue_metrics_*",
+  "template": {
+    "settings": {
+      "number_of_replicas": 0,
+      "number_of_shards": 1
+    },
+    "mappings": {
+      "_source": {
+        "enabled": true
+      },
+      "properties": {
+        "timestamp": {
+          "type": "date"
+        },
+        "queue": {
+          "type": "keyword"
+        },
+        "average_waiting_time": {
+          "type": "float"
+        },
+        "queue_length": {
+          "type": "integer"
+        },
+        "company_id": {
+          "type": "keyword"
+        }
+      }
+    }
+  }
+}

apiserver/elastic/index_templates/workers/serving_stats.json

@@ -0,0 +1,79 @@
+{
+  "index_patterns": "serving_stats_*",
+  "template": {
+    "settings": {
+      "number_of_replicas": 0,
+      "number_of_shards": 1
+    },
+    "mappings": {
+      "_source": {
+        "enabled": true
+      },
+      "properties": {
+        "timestamp": {
+          "type": "date"
+        },
+        "container_id": {
+          "type": "keyword"
+        },
+        "company_id": {
+          "type": "keyword"
+        },
+        "endpoint_url": {
+          "type": "keyword"
+        },
+        "requests_num": {
+          "type": "integer"
+        },
+        "requests_min": {
+          "type": "float"
+        },
+        "uptime_sec": {
+          "type": "integer"
+        },
+        "latency_ms": {
+          "type": "integer"
+        },
+        "cpu_usage": {
+          "type": "float"
+        },
+        "cpu_num": {
+          "type": "integer"
+        },
+        "gpu_usage": {
+          "type": "float"
+        },
+        "gpu_num": {
+          "type": "integer"
+        },
+        "memory_used": {
+          "type": "float"
+        },
+        "memory_free": {
+          "type": "float"
+        },
+        "memory_total": {
+          "type": "float"
+        },
+        "gpu_memory_used": {
+          "type": "float"
+        },
+        "gpu_memory_free": {
+          "type": "float"
+        },
+        "gpu_memory_total": {
+          "type": "float"
+        },
+        "disk_free_home": {
+          "type": "float"
+        },
+        "network_rx": {
+          "type": "float"
+        },
+        "network_tx": {
+          "type": "float"
+        }
+      }
+    }
+  }
+}

apiserver/elastic/index_templates/workers/worker_stats.json

@@ -0,0 +1,43 @@
+{
+  "index_patterns": "worker_stats_*",
+  "template": {
+    "settings": {
+      "number_of_replicas": 0,
+      "number_of_shards": 1
+    },
+    "mappings": {
+      "_source": {
+        "enabled": true
+      },
+      "properties": {
+        "timestamp": {
+          "type": "date"
+        },
+        "worker": {
+          "type": "keyword"
+        },
+        "category": {
+          "type": "keyword"
+        },
+        "metric": {
+          "type": "keyword"
+        },
+        "variant": {
+          "type": "keyword"
+        },
+        "value": {
+          "type": "float"
+        },
+        "unit": {
+          "type": "keyword"
+        },
+        "task": {
+          "type": "keyword"
+        },
+        "company_id": {
+          "type": "keyword"
+        }
+      }
+    }
+  }
+}

apiserver/elastic/initialize.py

@@ -10,6 +10,8 @@ from apiserver.config_repo import config
 from apiserver.elastic.apply_mappings import apply_mappings_to_cluster
 
 log = config.logger(__file__)
+logging.getLogger("elasticsearch").setLevel(logging.WARNING)
+logging.getLogger("elastic_transport").setLevel(logging.WARNING)
 
 
 class MissingElasticConfiguration(Exception):
@@ -78,6 +80,18 @@ def check_elastic_empty() -> bool:
         err_type=urllib3.exceptions.NewConnectionError, args_prefix=("GET",)
     )
 
+    def events_legacy_template():
+        try:
+            return es.indices.get_template(name="events*")
+        except exceptions.NotFoundError:
+            return False
+
+    def events_template():
+        try:
+            return es.indices.get_index_template(name="events*")
+        except exceptions.NotFoundError:
+            return False
+
     try:
         es_logger.addFilter(log_filter)
         for retry in range(max_retries):
@@ -87,10 +101,7 @@ def check_elastic_empty() -> bool:
                     http_auth=es_factory.get_credentials("events", cluster_conf),
                     **cluster_conf.get("args", {}),
                 )
-                return not es.indices.get_template(name="events*")
-            except exceptions.NotFoundError as ex:
-                log.error(ex)
-                return True
+                return not (events_template() or events_legacy_template())
             except exceptions.ConnectionError as ex:
                 if retry >= max_retries - 1:
                     raise ElasticConnectionError(

apiserver/es_factory.py

@@ -1,3 +1,4 @@
+import logging
 from datetime import datetime
 from functools import lru_cache
 from os import getenv
@@ -9,6 +10,8 @@ from elasticsearch import Elasticsearch
 from apiserver.config_repo import config
 
 log = config.logger(__file__)
+logging.getLogger('elasticsearch').setLevel(logging.WARNING)
+logging.getLogger('elastic_transport').setLevel(logging.WARNING)
 
 OVERRIDE_HOST_ENV_KEY = (
     "CLEARML_ELASTIC_SERVICE_HOST",
@@ -32,6 +35,7 @@ if OVERRIDE_HOST:
 
 OVERRIDE_PORT = first(filter(None, map(getenv, OVERRIDE_PORT_ENV_KEY)))
 if OVERRIDE_PORT:
+    OVERRIDE_PORT = int(OVERRIDE_PORT)
     log.info(f"Using override elastic port {OVERRIDE_PORT}")
 
 OVERRIDE_USERNAME = first(filter(None, map(getenv, OVERRIDE_USERNAME_ENV_KEY)))

apiserver/fix_mongo_urls.py

@@ -0,0 +1,122 @@
+import logging
+from argparse import (
+    ArgumentDefaultsHelpFormatter,
+    ArgumentParser,
+    ArgumentTypeError,
+)
+
+from pymongo import MongoClient
+from pymongo.collection import Collection
+from pymongo.database import Database
+
+
+logging.getLogger().setLevel(logging.INFO)
+
+
+def fix_mongo_urls(mongo_host: str, host_source: str, host_target: str):
+    logging.info(f"Connecting to Mongo on {mongo_host}")
+    client = MongoClient(host=mongo_host)
+    backend_db: Database = client.backend
+
+    def get_updated_uri(uri: str):
+        if not uri or not uri.startswith(host_source):
+            return
+        relative_url = uri[len(host_source) :]
+        return f"{host_target.rstrip('/')}/{relative_url.lstrip('/')}"
+
+    host_source = host_source
+    host_target = host_target
+    model_collection: Collection = backend_db.get_collection("model")
+    if model_collection is not None:
+        logging.info("Updating model uris")
+        models_count = model_collection.count_documents({})
+        updated_models = 0
+        for model in model_collection.find(
+            {"uri": {"$regex": "^{}".format(host_source)}}, projection=["uri"]
+        ):
+            updated_uri = get_updated_uri(model.get("uri"))
+            if updated_uri:
+                result = model_collection.update_one(
+                    {"_id": model["_id"]}, {"$set": {"uri": updated_uri}}
+                )
+                updated_models += result.modified_count
+
+        logging.info(f"Updated {updated_models} models from {models_count}")
+
+    task_collection: Collection = backend_db.get_collection("task")
+    if task_collection is not None:
+        logging.info("Updating task uris")
+        tasks_count = task_collection.count_documents({})
+        updated_tasks = 0
+        for task in task_collection.find(
+            {"execution.artifacts": {"$exists": 1, "$ne": {}}},
+            projection=["execution.artifacts"],
+        ):
+            artifacts = task.get("execution", {}).get("artifacts")
+            if not artifacts:
+                continue
+
+            uri_updated = False
+            for artifact in artifacts.values():
+                updated_uri = get_updated_uri(artifact.get("uri"))
+                if updated_uri:
+                    artifact["uri"] = updated_uri
+                    uri_updated = True
+
+            if uri_updated:
+                result = task_collection.update_one(
+                    {"_id": task["_id"]}, {"$set": {"execution.artifacts": artifacts}}
+                )
+                updated_tasks += result.modified_count
+
+        logging.info(f"Updated {updated_tasks} tasks from {tasks_count}")
+
+
+def normalise_host(host):
+    if not host.endswith("/"):
+        return host
+    return host[:-1]
+
+
+def main():
+    def valid_url_prefix(url: str):
+        if "://" not in url:
+            raise ArgumentTypeError("url schema is missing")
+        return url
+
+    parser = ArgumentParser(
+        description=__doc__, formatter_class=ArgumentDefaultsHelpFormatter
+    )
+    parser.add_argument(
+        "--mongo-host",
+        "-mh",
+        type=str,
+        default="mongodb://mongo:27017",
+        help="Mongo server host. The default is mongodb://mongo:27017",
+    )
+    parser.add_argument(
+        "--host-source",
+        "-hs",
+        type=valid_url_prefix,
+        required=True,
+        help="Source host for the files uploaded to the fileserver (in the form http://<host>:<port>)",
+    )
+    parser.add_argument(
+        "--host-target",
+        "-ht",
+        type=valid_url_prefix,
+        required=True,
+        help="Target host for the files uploaded to the fileserver (in the form http://<host>:<port>)",
+    )
+    args = parser.parse_args()
+
+    fix_mongo_urls(
+        mongo_host=args.mongo_host,
+        host_source=args.host_source,
+        host_target=args.host_target,
+    )
+    logging.info("Completed successfully")
+
+
+if __name__ == "__main__":
+    main()

apiserver/jobs/async_urls_delete.py

@@ -19,7 +19,9 @@ from google.cloud import storage as google_storage
 from mongoengine import Q
 from mypy_boto3_s3.service_resource import Bucket as AWSBucket
 
+from apiserver.bll.auth import AuthBLL
 from apiserver.bll.storage import StorageBLL
+from apiserver.config.info import get_default_company
 from apiserver.config_repo import config
 from apiserver.database import db
 from apiserver.database.model.url_to_delete import UrlToDelete, StorageType, DeletionStatus
@@ -200,6 +202,8 @@ class FileserverStorage(Storage):
             res_data = res.json()
             return list(res_data.get("deleted", {})), res_data.get("errors", {})
 
+    token_expiration_sec = conf.get("fileserver.token_expiration_sec", 600)
+
     def __init__(self, company: str, fileserver_host: str = None):
         fileserver_host = fileserver_host or config.get("hosts.fileserver", None)
         self.host = fileserver_host.rstrip("/")
@@ -220,13 +224,6 @@ class FileserverStorage(Storage):
 
         self.company = company
 
-    # @classmethod
-    # def validate_fileserver_access(cls, fileserver_host: str):
-    #     res = requests.get(
-    #         url=fileserver_host
-    #     )
-    #     res.raise_for_status()
-
     @property
     def name(self) -> str:
         return "Fileserver"
@@ -260,7 +257,13 @@ class FileserverStorage(Storage):
 
     def get_client(self, base: str, urls: Sequence[UrlToDelete]) -> Client:
         host = base
+        token = AuthBLL.get_token_for_user(
+            user_id="__apiserver__",
+            company_id=get_default_company(),
+            expiration_sec=self.token_expiration_sec,
+        ).token
         session = requests.session()
+        session.headers.update({"Authorization": "Bearer {}".format(token)})
         res = session.get(url=host, timeout=self.Client.timeout)
         res.raise_for_status()
 
@@ -285,6 +288,7 @@ class AzureStorage(Storage):
             ):
                 raise ValueError("No path found following container name")
 
+            # noinspection PyTypeChecker
             return os.path.join(*parsed.path.segments[1:])
 
         @staticmethod
@@ -450,6 +454,7 @@ class AWSStorage(Storage):
             else None,
             "use_ssl": cfg.secure,
             "verify": cfg.verify,
+            "region_name": cfg.region or None,
         }
         name = base[len(scheme_prefix(self.scheme)) :]
         bucket_name = name[len(cfg.host) + 1 :] if cfg.host else name

apiserver/mongo/initialize/__init__.py

@@ -3,7 +3,7 @@ from typing import Sequence, Union
 
 from apiserver.config_repo import config
 from apiserver.config.info import get_default_company
-from apiserver.database.model.auth import Role
+from apiserver.database.model.auth import Role, User as AuthUser
 from apiserver.service_repo.auth.fixed_user import FixedUser
 from .migration import _apply_migrations, check_mongo_empty, get_last_server_version
 from .pre_populate import PrePopulate
@@ -60,16 +60,20 @@ def init_mongo_data():
 
         fixed_mode = FixedUser.enabled()
 
+        internal_user_emails = set()
         for user, credentials in config.get("secure.credentials", {}).items():
+            email = f"{user}@example.com"
             user_data = {
                 "name": user,
                 "role": credentials.role,
-                "email": f"{user}@example.com",
+                "email": email,
                 "key": credentials.user_key,
                 "secret": credentials.user_secret,
+                "autocreated": True,
             }
+            internal_user_emails.add(email.lower())
             revoke = fixed_mode and credentials.get("revoke_in_fixed_mode", False)
-            user_id = _ensure_auth_user(user_data, company_id, log=log, revoke=revoke)
+            user_id = _ensure_auth_user(user_data, company_id, log=log, revoke=revoke, internal_user=True)
             if credentials.role == Role.user:
                 _ensure_backend_user(user_id, company_id, credentials.display_name)
 
@@ -82,8 +86,20 @@ def init_mongo_data():
 
             for user in FixedUser.from_config():
                 try:
-                    ensure_fixed_user(user, log=log)
+                    ensure_fixed_user(user, log=log, emails=internal_user_emails)
                 except Exception as ex:
                     log.error(f"Failed creating fixed user {user.name}: {ex}")
+
+        if internal_user_emails and config.get(
+            f"apiserver.auth.delete_missing_autocreated_users", True
+        ):
+            for user in AuthUser.objects(
+                company=company_id, autocreated=True, email__nin=internal_user_emails
+            ):
+                log.info(
+                    f"Removing user that is no longer in configuration: {user['id']}\t{user['email']}\t{user['name']}"
+                )
+                user.delete()
+
     except Exception as ex:
-        log.exception("Failed initializing mongodb")
+        log.exception(f"Failed initializing mongodb: {str(ex)}")

apiserver/mongo/initialize/migration.py

@@ -8,13 +8,16 @@ import pymongo.database
 from mongoengine.connection import get_db
 from packaging.version import Version, parse
 
+from apiserver.config_repo import config
 from apiserver.database import utils
 from apiserver.database import Database
 from apiserver.database.model.version import Version as DatabaseVersion
+from apiserver.utilities.dicts import nested_get
 
 _migrations = "migrations"
 _parent_dir = Path(__file__).resolve().parents[1]
 _migration_dir = _parent_dir / _migrations
+log = config.logger(__file__)
 
 
 def check_mongo_empty() -> bool:
@@ -41,6 +44,26 @@ def get_last_server_version() -> Version:
     return previous_versions[0] if previous_versions else Version("0.0.0")
 
 
+def _ensure_mongodb_version():
+    db: pymongo.database.Database = get_db(Database.backend)
+    db_version = db.client.server_info()["version"]
+    if not db_version.startswith("5.0"):
+        log.warning(f"Database version should be 5.0.x. Instead: {str(db_version)}")
+        return
+
+    res = db.client.admin.command({"getParameter": 1, "featureCompatibilityVersion": 1})
+    version = nested_get(res, ("featureCompatibilityVersion", "version"))
+    if version == "5.0":
+        return
+    if version != "4.4":
+        log.warning(f"Cannot upgrade DB version. Should be 4.4. {str(res)}")
+        return
+
+    log.info("Upgrading db version from 4.4 to 5.0")
+    res = db.client.admin.command({"setFeatureCompatibilityVersion": "5.0"})
+    log.info(res)
+
+
 def _apply_migrations(log: Logger):
     """
     Apply migrations as found in the migration dir.
@@ -50,6 +73,8 @@ def _apply_migrations(log: Logger):
 
     log.info(f"Started mongodb migrations")
 
+    _ensure_mongodb_version()
+
     if not _migration_dir.is_dir():
         raise ValueError(f"Invalid migration dir {_migration_dir}")
 

apiserver/mongo/initialize/pre_populate.py

@@ -22,15 +22,17 @@ from typing import (
     Mapping,
     IO,
     Callable,
+    Iterable,
 )
 from urllib.parse import unquote, urlparse
 from uuid import uuid4, UUID, uuid5
 from zipfile import ZipFile, ZIP_BZIP2
 
+import attr
 import mongoengine
 from boltons.iterutils import chunked_iter, first
 from furl import furl
-from mongoengine import Q
+from mongoengine import Q, Document
 
 from apiserver.bll.event import EventBLL
 from apiserver.bll.event.event_common import EventType
@@ -44,6 +46,7 @@ from apiserver.bll.task.param_utils import (
 from apiserver.config_repo import config
 from apiserver.config.info import get_default_company
 from apiserver.database.model import EntityVisibility, User
+from apiserver.database.model.auth import Role, User as AuthUser
 from apiserver.database.model.model import Model
 from apiserver.database.model.project import Project
 from apiserver.database.model.task.task import (
@@ -54,10 +57,13 @@ from apiserver.database.model.task.task import (
     TaskModelNames,
 )
 from apiserver.database.utils import get_options
+from apiserver.service_repo.auth import Identity
 from apiserver.utilities import json
 from apiserver.utilities.dicts import nested_get, nested_set, nested_delete
 from apiserver.utilities.parameter_key_escaper import ParameterKeyEscaper
 
+replace_s3_scheme = os.getenv("CLEARML_REPLACE_S3_SCHEME")
+
 
 class PrePopulate:
     module_name_prefix = "apiserver."
@@ -66,6 +72,7 @@ class PrePopulate:
     export_tag_prefix = "Exported:"
     export_tag = f"{export_tag_prefix} %Y-%m-%d %H:%M:%S"
     metadata_filename = "metadata.json"
+    users_filename = "users.json"
     zip_args = dict(mode="w", compression=ZIP_BZIP2)
     artifacts_ext = ".artifacts"
     img_source_regex = re.compile(
@@ -78,6 +85,12 @@ class PrePopulate:
     project_cls: Type[Project]
     model_cls: Type[Model]
     user_cls: Type[User]
+    auth_user_cls: Type[AuthUser]
+
+    @attr.s(auto_attribs=True)
+    class ParentPrefix:
+        prefix: str
+        path: Sequence[str]
 
     # noinspection PyTypeChecker
     @classmethod
@@ -90,6 +103,8 @@ class PrePopulate:
             cls.project_cls = cls._get_entity_type("database.model.project.Project")
         if not hasattr(cls, "user_cls"):
             cls.user_cls = cls._get_entity_type("database.model.User")
+        if not hasattr(cls, "auth_user_cls"):
+            cls.auth_user_cls = cls._get_entity_type("database.model.auth.User")
 
     class JsonLinesWriter:
         def __init__(self, file: BinaryIO):
@@ -205,6 +220,8 @@ class PrePopulate:
         task_statuses: Sequence[str] = None,
         tag_exported_entities: bool = False,
         metadata: Mapping[str, Any] = None,
+        export_events: bool = True,
+        export_users: bool = False,
     ) -> Sequence[str]:
         cls._init_entity_types()
 
@@ -212,6 +229,9 @@ class PrePopulate:
             raise ValueError("Invalid task statuses")
 
         file = Path(filename)
+        if not (experiments or projects):
+            projects = cls.project_cls.objects(parent=None).scalar("id")
+
         entities = cls._resolve_entities(
             experiments=experiments, projects=projects, task_statuses=task_statuses
         )
@@ -240,11 +260,15 @@ class PrePopulate:
         with ZipFile(file, **cls.zip_args) as zfile:
             if metadata:
                 zfile.writestr(cls.metadata_filename, meta_str)
+            if export_users:
+                cls._export_users(zfile)
             artifacts = cls._export(
                 zfile,
                 entities=entities,
                 hash_=hash_,
                 tag_entities=tag_exported_entities,
+                export_events=export_events,
+                cleanup_users=not export_users,
             )
 
         file_with_hash = file.with_name(f"{file.stem}_{hash_.hexdigest()}{file.suffix}")
@@ -265,6 +289,9 @@ class PrePopulate:
             metadata_hash=metadata_hash,
         )
 
+        if created_files:
+            print("Created files:\n" + "\n".join(file for file in created_files))
+
         return created_files
 
     @classmethod
@@ -296,18 +323,26 @@ class PrePopulate:
             except Exception:
                 pass
 
-            if not user_id:
-                user_id, user_name = "__allegroai__", "Allegro.ai"
-
             # Make sure we won't end up with an invalid company ID
             if company_id is None:
                 company_id = ""
 
+            user_mapping = cls._import_users(zfile, company_id)
+
+            if not user_id:
+                user_id, user_name = "__allegroai__", "Allegro.ai"
+
             existing_user = cls.user_cls.objects(id=user_id).only("id").first()
             if not existing_user:
                 cls.user_cls(id=user_id, name=user_name, company=company_id).save()
 
-            cls._import(zfile, company_id, user_id, metadata)
+            cls._import(
+                zfile,
+                company_id=company_id,
+                user_id=user_id,
+                metadata=metadata,
+                user_mapping=user_mapping,
+            )
 
         if artifacts_path and os.path.isdir(artifacts_path):
             artifacts_file = Path(filename).with_suffix(cls.artifacts_ext)
@@ -394,42 +429,83 @@ class PrePopulate:
             featured_index = get_index(project)
             cls.project_cls.objects(id=project.id).update(featured=featured_index)
 
-    @staticmethod
-    def _resolve_type(
-        cls: Type[mongoengine.Document], ids: Optional[Sequence[str]]
+    @classmethod
+    def _resolve_entity_type(
+        cls, entity_type: Type[mongoengine.Document], ids: Optional[Sequence[str]]
     ) -> Sequence[Any]:
         ids = set(ids)
-        items = list(cls.objects(id__in=list(ids)))
+        items = list(entity_type.objects(id__in=list(ids)))
         resolved = {i.id for i in items}
         missing = ids - resolved
-        for name_candidate in missing:
-            results = list(cls.objects(name=name_candidate))
-            if not results:
-                print(f"ERROR: no match for `{name_candidate}`")
-                exit(1)
-            elif len(results) > 1:
-                print(f"ERROR: more than one match for `{name_candidate}`")
-                exit(1)
-            items.append(results[0])
-        return items
+        if not missing:
+            return items
+
+        resolved_by_name = defaultdict(list)
+        for entity in entity_type.objects(name__in=list(missing)):
+            resolved_by_name[entity.name].append(entity)
+
+        not_found = missing - set(resolved_by_name)
+        if not_found:
+            print(f"ERROR: no match for {', '.join(not_found)}")
+            exit(1)
+
+        duplicates = [k for k, v in resolved_by_name.items() if len(v) > 1]
+        if duplicates:
+            print(f"ERROR: more than one match for {', '.join(duplicates)}")
+            exit(1)
+
+        def get_new_items(input_: Iterable) -> list:
+            return [item for item in input_ if item.id not in resolved]
+
+        def get_projects_with_children(projects: list) -> list:
+            project_ids = set(item.id for item in projects)
+            ids_with_children = project_ids_with_children(list(project_ids))
+            if project_ids == set(ids_with_children):
+                return projects
+
+            return get_new_items(entity_type.objects(id__in=ids_with_children))
+
+        new_items = get_new_items(chain(*resolved_by_name.values()))
+        if not new_items:
+            return items
+
+        if entity_type == cls.project_cls:
+            new_items = get_projects_with_children(new_items)
+
+        return items + new_items
 
     @classmethod
     def _check_projects_hierarchy(cls, projects: Set[Project]):
         """
-        For any exported project all its parents up to the root should be present
+        For the projects that are exported not from the root
+        fix their parents tree to exclude the not exported parents
         """
         if not projects:
             return
 
         project_ids = {p.id for p in projects}
-        orphans = [p.id for p in projects if p.parent and p.parent not in project_ids]
+        orphans = [p for p in projects if p.parent and p.parent not in project_ids]
         if not orphans:
             return
 
-        print(
-            f"ERROR: the following projects are exported without their parents: {orphans}"
-        )
-        exit(1)
+        prefixes = [
+            cls.ParentPrefix(prefix=f"{project.name.rpartition('/')[0]}/", path=project.path)
+            for project in orphans
+        ]
+        prefixes.sort(key=lambda p: len(p.path), reverse=True)
+        for project in projects:
+            prefix = first(pref for pref in prefixes if project.path[:len(pref.path)] == pref.path)
+            if not prefix:
+                continue
+            project.path = project.path[len(prefix.path):]
+            if not project.path:
+                project.parent = None
+            project.name = project.name.removeprefix(prefix.prefix)
+
+        # print(
+        #     f"ERROR: the following projects are exported without their parents: {orphans}"
+        # )
+        # exit(1)
 
     @classmethod
     def _resolve_entities(
@@ -438,13 +514,14 @@ class PrePopulate:
         projects: Sequence[str] = None,
         task_statuses: Sequence[str] = None,
     ) -> Dict[Type[mongoengine.Document], Set[mongoengine.Document]]:
-        entities = defaultdict(set)
+        # noinspection PyTypeChecker
+        entities: Dict[Any] = defaultdict(set)
 
         if projects:
             print("Reading projects...")
             projects = project_ids_with_children(projects)
             entities[cls.project_cls].update(
-                cls._resolve_type(cls.project_cls, projects)
+                cls._resolve_entity_type(cls.project_cls, projects)
             )
             print("--> Reading project experiments...")
             query = Q(
@@ -462,7 +539,7 @@ class PrePopulate:
 
         if experiments:
             print("Reading experiments...")
-            entities[cls.task_cls].update(cls._resolve_type(cls.task_cls, experiments))
+            entities[cls.task_cls].update(cls._resolve_entity_type(cls.task_cls, experiments))
             print("--> Reading experiments projects...")
             objs = cls.project_cls.objects(
                 id__in=list(
@@ -486,6 +563,7 @@ class PrePopulate:
             print("Reading models...")
             entities[cls.model_cls] = set(cls.model_cls.objects(id__in=list(model_ids)))
 
+        # noinspection PyTypeChecker
         return entities
 
     @classmethod
@@ -497,7 +575,6 @@ class PrePopulate:
     @classmethod
     def _cleanup_model(cls, model: Model):
         model.company = ""
-        model.user = ""
         model.tags = cls._filter_out_export_tags(model.tags)
 
     @classmethod
@@ -505,7 +582,6 @@ class PrePopulate:
         task.comment = "Auto generated by Allegro.ai"
         task.status_message = ""
         task.status_reason = ""
-        task.user = ""
         task.company = ""
         task.tags = cls._filter_out_export_tags(task.tags)
         if task.output:
@@ -513,17 +589,32 @@ class PrePopulate:
 
     @classmethod
     def _cleanup_project(cls, project: Project):
-        project.user = ""
         project.company = ""
         project.tags = cls._filter_out_export_tags(project.tags)
 
     @classmethod
-    def _cleanup_entity(cls, entity_cls, entity):
+    def _cleanup_auth_user(cls, user: AuthUser):
+        user.company = ""
+        for cred in user.credentials:
+            if getattr(cred, "company", None):
+                cred["company"] = ""
+        return user
+
+    @classmethod
+    def _cleanup_be_user(cls, user: User):
+        user.company = ""
+        user.preferences = None
+        return user
+
+    @classmethod
+    def _cleanup_entity(cls, entity_cls, entity, cleanup_users):
+        if cleanup_users:
+            entity.user = ""
         if entity_cls == cls.task_cls:
             cls._cleanup_task(entity)
         elif entity_cls == cls.model_cls:
             cls._cleanup_model(entity)
-        elif entity == cls.project_cls:
+        elif entity_cls == cls.project_cls:
             cls._cleanup_project(entity)
 
     @classmethod
@@ -577,8 +668,9 @@ class PrePopulate:
 
     @staticmethod
     def _get_fixed_url(url: Optional[str]) -> Optional[str]:
-        if not (url and url.lower().startswith("s3://")):
+        if not (replace_s3_scheme and url and url.lower().startswith("s3://")):
             return url
+
         try:
             fixed = furl(url)
             fixed.scheme = "https"
@@ -633,6 +725,38 @@ class PrePopulate:
             else:
                 print(f"Artifact {full_path} not found")
 
+    @classmethod
+    def _export_users(cls, writer: ZipFile):
+        auth_users = {
+            user.id: cls._cleanup_auth_user(user)
+            for user in cls.auth_user_cls.objects(role__in=(Role.admin, Role.user))
+        }
+        if not auth_users:
+            return
+
+        be_users = {
+            user.id: cls._cleanup_be_user(user)
+            for user in cls.user_cls.objects(id__in=list(auth_users))
+        }
+        if not be_users:
+            return
+
+        auth_users = {uid: data for uid, data in auth_users.items() if uid in be_users}
+        print(f"Writing {len(auth_users)} users into {writer.filename}")
+        data = {}
+        for field, users in (("auth", auth_users), ("backend", be_users)):
+            with BytesIO() as f:
+                with cls.JsonLinesWriter(f) as w:
+                    for user in users.values():
+                        w.write(user.to_json())
+                data[field] = f.getvalue()
+
+        def get_field_bytes(k: str, v: bytes) -> bytes:
+            return f'"{k}": '.encode("utf-8") + v
+
+        data_str = b",\n".join(get_field_bytes(k, v) for k, v in data.items())
+        writer.writestr(cls.users_filename, b"{\n" + data_str + b"\n}")
+
     @classmethod
     def _get_base_filename(cls, cls_: type):
         name = f"{cls_.__module__}.{cls_.__name__}"
@@ -642,7 +766,13 @@ class PrePopulate:
 
     @classmethod
     def _export(
-        cls, writer: ZipFile, entities: dict, hash_, tag_entities: bool = False
+        cls,
+        writer: ZipFile,
+        entities: dict,
+        hash_,
+        tag_entities: bool = False,
+        export_events: bool = True,
+        cleanup_users: bool = True,
     ) -> Sequence[str]:
         """
         Export the requested experiments, projects and models and return the list of artifact files
@@ -656,18 +786,19 @@ class PrePopulate:
             if not items:
                 continue
             base_filename = cls._get_base_filename(cls_)
-            for item in items:
-                artifacts.extend(
-                    cls._export_entity_related_data(
-                        cls_, item, base_filename, writer, hash_
+            if export_events:
+                for item in items:
+                    artifacts.extend(
+                        cls._export_entity_related_data(
+                            cls_, item, base_filename, writer, hash_
+                        )
                     )
-                )
             filename = base_filename + ".json"
             print(f"Writing {len(items)} items into {writer.filename}:{filename}")
             with BytesIO() as f:
                 with cls.JsonLinesWriter(f) as w:
                     for item in items:
-                        cls._cleanup_entity(cls_, item)
+                        cls._cleanup_entity(cls_, item, cleanup_users=cleanup_users)
                         w.write(item.to_json())
                 data = f.getvalue()
                 hash_.update(data)
@@ -717,7 +848,10 @@ class PrePopulate:
 
     @classmethod
     def _generate_new_ids(
-        cls, reader: ZipFile, entity_files: Sequence, metadata: Mapping[str, Any],
+        cls,
+        reader: ZipFile,
+        entity_files: Sequence,
+        metadata: Mapping[str, Any],
     ) -> Mapping[str, str]:
         if not metadata or not any(
             metadata.get(key) for key in ("new_ids", "example_ids", "private_ids")
@@ -745,6 +879,68 @@ class PrePopulate:
                         )
         return ids
 
+    @classmethod
+    def _import_users(cls, reader: ZipFile, company_id: str = "") -> dict:
+        """
+        Import users to db and return the mapping of old user ids to the new ones
+        If no users were in the users file then the mapping was empty
+        If the user in the file has the same email as one of the existing ones then this user is skipped
+        and its id is mapped to the existing user with the same email
+        If the user with the same id exists in backend or auth db then its creation is skipped
+        """
+        users_file = first(
+            fi for fi in reader.filelist if fi.orig_filename == cls.users_filename
+        )
+        if not users_file:
+            return {}
+
+        existing_user_ids = set(cls.user_cls.objects().scalar("id")) | set(
+            cls.auth_user_cls.objects().scalar("id")
+        )
+        existing_user_emails = {u.email: u.id for u in cls.auth_user_cls.objects()}
+        user_id_mappings = {}
+
+        with reader.open(users_file) as f:
+            data = json.loads(f.read())
+
+        auth_users = {u["_id"]: u for u in data["auth"]}
+        be_users = {u["_id"]: u for u in data["backend"]}
+        for uid, user in auth_users.items():
+            email = user.get("email")
+            existing_user_id = existing_user_emails.get(email)
+            if existing_user_id:
+                user_id_mappings[uid] = existing_user_id
+                continue
+
+            user_id_mappings[uid] = uid
+            if uid in existing_user_ids:
+                continue
+
+            credentials = user.get("credentials", [])
+            for c in credentials:
+                if c.get("company") == "":
+                    c["company"] = company_id
+
+            if hasattr(cls.auth_user_cls, "sec_groups"):
+                user_role = user.get("role", Role.user)
+                if user_role == Role.user:
+                    user["sec_groups"] = ["30795571-a470-4717-a80d-e8705fc776bf"]
+                else:
+                    user["sec_groups"] = [
+                        "c14a3cc6-1144-4896-8ea6-fb186ee19896",
+                        "30795571-a470-4717-a80d-e8705fc776bf",
+                        "30795571a4704717a80de8705897ytuyg",
+                    ]
+
+            auth_user = cls.auth_user_cls.from_json(json.dumps(user), created=True)
+            auth_user.company = company_id
+            auth_user.save()
+            be_user = cls.user_cls.from_json(json.dumps(be_users[uid]), created=True)
+            be_user.company = company_id
+            be_user.save()
+
+        return user_id_mappings
+
     @classmethod
     def _import(
         cls,
@@ -753,6 +949,7 @@ class PrePopulate:
         user_id: str = None,
         metadata: Mapping[str, Any] = None,
         sort_tasks_by_last_updated: bool = True,
+        user_mapping: Mapping[str, str] = None,
     ):
         """
         Import entities and events from the zip file
@@ -763,7 +960,7 @@ class PrePopulate:
             fi
             for fi in reader.filelist
             if not fi.orig_filename.endswith(event_file_ending)
-            and fi.orig_filename != cls.metadata_filename
+            and fi.orig_filename not in (cls.metadata_filename, cls.users_filename)
         ]
         metadata = metadata or {}
         old_to_new_ids = cls._generate_new_ids(reader, entity_files, metadata)
@@ -773,7 +970,13 @@ class PrePopulate:
                 full_name = splitext(entity_file.orig_filename)[0]
                 print(f"Reading {reader.filename}:{full_name}...")
                 res = cls._import_entity(
-                    f, full_name, company_id, user_id, metadata, old_to_new_ids
+                    f,
+                    full_name=full_name,
+                    company_id=company_id,
+                    user_id=user_id,
+                    metadata=metadata,
+                    old_to_new_ids=old_to_new_ids,
+                    user_mapping=user_mapping,
                 )
                 if res:
                     tasks = res
@@ -794,7 +997,7 @@ class PrePopulate:
             with reader.open(events_file) as f:
                 full_name = splitext(events_file.orig_filename)[0]
                 print(f"Reading {reader.filename}:{full_name}...")
-                cls._import_events(f, company_id, user_id, task.id)
+                cls._import_events(f, company_id, task.user, task.id)
 
     @classmethod
     def _get_entity_type(cls, full_name) -> Type[mongoengine.Document]:
@@ -806,8 +1009,10 @@ class PrePopulate:
         module = importlib.import_module(module_name)
         return getattr(module, class_name)
 
-    @staticmethod
-    def _upgrade_project_data(project_data: dict) -> dict:
+    @classmethod
+    def _upgrade_project_data(cls, project_data: dict) -> dict:
+        cls._remove_incompatible_fields(cls.project_cls, project_data)
+
         if not project_data.get("basename"):
             name: str = project_data["name"]
             _, _, basename = name.rpartition("/")
@@ -815,8 +1020,10 @@ class PrePopulate:
 
         return project_data
 
-    @staticmethod
-    def _upgrade_model_data(model_data: dict) -> dict:
+    @classmethod
+    def _upgrade_model_data(cls, model_data: dict) -> dict:
+        cls._remove_incompatible_fields(cls.model_cls, model_data)
+
         metadata_key = "metadata"
         metadata = model_data.get(metadata_key)
         if isinstance(metadata, list):
@@ -829,7 +1036,13 @@ class PrePopulate:
         return model_data
 
     @staticmethod
-    def _upgrade_task_data(task_data: dict) -> dict:
+    def _remove_incompatible_fields(cls_: Type[Document], data: dict):
+        for field in ("company_origin",):
+            if field not in cls_._db_field_map:
+                data.pop(field, None)
+
+    @classmethod
+    def _upgrade_task_data(cls, task_data: dict) -> dict:
         """
         Migrate from execution/parameters and model_desc to hyperparams and configuration fiields
         Upgrade artifacts list to dict
@@ -838,6 +1051,8 @@ class PrePopulate:
         :param task_data: Upgraded in place
         :return: The upgraded task data
         """
+        cls._remove_incompatible_fields(cls.task_cls, task_data)
+
         for old_param_field, new_param_field, default_section in (
             ("execution.parameters", "hyperparams", hyperparams_default_section),
             ("execution.model_desc", "configuration", None),
@@ -874,7 +1089,7 @@ class PrePopulate:
         ):
             old_path = old_field.split(".")
             old_model = nested_get(task_data, old_path)
-            new_models = models.get(type_, [])
+            new_models = [m for m in models.get(type_, []) if m.get("model") is not None]
             name = TaskModelNames[type_]
             if old_model and not any(
                 m
@@ -908,7 +1123,9 @@ class PrePopulate:
         user_id: str,
         metadata: Mapping[str, Any],
         old_to_new_ids: Mapping[str, str] = None,
+        user_mapping: Mapping[str, str] = None,
     ) -> Optional[Sequence[Task]]:
+        user_mapping = user_mapping or {}
         cls_ = cls._get_entity_type(full_name)
         print(f"Writing {cls_.__name__.lower()}s into database")
         tasks = []
@@ -930,7 +1147,7 @@ class PrePopulate:
 
             doc = cls_.from_json(item, created=True)
             if hasattr(doc, "user"):
-                doc.user = user_id
+                doc.user = user_mapping.get(doc.user, user_id) if doc.user else user_id
             if hasattr(doc, "company"):
                 doc.company = company_id
             if isinstance(doc, cls.project_cls):
@@ -954,13 +1171,13 @@ class PrePopulate:
 
             if isinstance(doc, cls.task_cls):
                 tasks.append(doc)
-                cls.event_bll.delete_task_events(company_id, doc.id, allow_locked=True)
+                cls.event_bll.delete_task_events(company_id, doc.id)
 
         if tasks:
             return tasks
 
     @classmethod
-    def _import_events(cls, f: IO[bytes], company_id: str, _, task_id: str):
+    def _import_events(cls, f: IO[bytes], company_id: str, user_id: str, task_id: str):
         print(f"Writing events for task {task_id} into database")
         for events_chunk in chunked_iter(cls.json_lines(f), 1000):
             events = [json.loads(item) for item in events_chunk]
@@ -969,5 +1186,8 @@ class PrePopulate:
                 ev["company_id"] = company_id
                 ev["allow_locked"] = True
             cls.event_bll.add_events(
-                company_id, events=events, worker=""
+                company_id=company_id,
+                identity=Identity(user_id, company=company_id, role=Role.admin),
+                events=events,
+                worker="",
             )

apiserver/mongo/initialize/user.py

@@ -9,34 +9,87 @@ from apiserver.database.model.user import User
 from apiserver.service_repo.auth.fixed_user import FixedUser
 
 
-def _ensure_auth_user(user_data: dict, company_id: str, log: Logger, revoke: bool = False):
-    key, secret = user_data.get("key"), user_data.get("secret")
+def _ensure_user_credentials(
+    user: AuthUser,
+    key: str,
+    secret: str,
+    log: Logger,
+    revoke: bool = False,
+    internal_user: bool = False,
+) -> None:
+    if revoke:
+        log.info(f"Revoking credentials for existing user {user.id} ({user.name})")
+        user.credentials = []
+        user.save()
+        return
+
     if not (key and secret):
-        credentials = None
-    else:
-        creds = Credentials(key=key, secret=secret)
+        if internal_user:
+            log.info(f"Resetting credentials for existing user {user.id} ({user.name})")
+            user.credentials = []
+            user.save()
+        return
 
-        user = AuthUser.objects(credentials__match=creds).first()
-        if user:
-            if revoke:
-                user.credentials = []
-                user.save()
-            return user.id
+    new_credentials = Credentials(key=key, secret=secret)
+    if internal_user:
+        log.info(f"Setting credentials for existing user {user.id} ({user.name})")
+        user.credentials = [new_credentials]
+        user.save()
+        return
 
-        credentials = [] if revoke else [creds]
+    if user.credentials is None:
+        user.credentials = []
+    if not any((cred.key, cred.secret) == (key, secret) for cred in user.credentials):
+        log.info(f"Adding credentials for existing user {user.id} ({user.name})")
+        user.credentials.append(new_credentials)
+        user.save()
 
+
+def _ensure_auth_user(
+    user_data: dict,
+    company_id: str,
+    log: Logger,
+    revoke: bool = False,
+    internal_user: bool = False,
+) -> str:
     user_id = user_data.get("id", f"__{user_data['name']}__")
+    role = user_data["role"]
+    email = user_data["email"]
+    autocreated = user_data.get("autocreated", False)
+    key, secret = user_data.get("key"), user_data.get("secret")
 
+    user: AuthUser = AuthUser.objects(id=user_id).first()
+    if user:
+        _ensure_user_credentials(
+            user=user,
+            key=key,
+            secret=secret,
+            log=log,
+            revoke=revoke,
+            internal_user=internal_user,
+        )
+        if user.role != role or user.email != email or user.autocreated != autocreated:
+            user.email = email
+            user.role = role
+            user.autocreated = autocreated
+            user.save()
+
+        return user.id
+
+    credentials = (
+        [Credentials(key=key, secret=secret)] if not revoke and key and secret else []
+    )
     log.info(f"Creating user: {user_data['name']}")
 
     user = AuthUser(
         id=user_id,
         name=user_data["name"],
         company=company_id,
-        role=user_data["role"],
-        email=user_data["email"],
+        role=role,
+        email=email,
         created=datetime.utcnow(),
         credentials=credentials,
+        autocreated=autocreated,
     )
 
     user.save()
@@ -59,23 +112,29 @@ def _ensure_backend_user(user_id: str, company_id: str, user_name: str):
     return user_id
 
 
-def ensure_fixed_user(user: FixedUser, log: Logger):
+def ensure_fixed_user(user: FixedUser, log: Logger, emails: set):
+    # noinspection PyTypeChecker
+    data = attr.asdict(user)
+    data["id"] = user.user_id
+    email = f"{user.user_id}@example.com"
+    data["email"] = email
+    data["role"] = Role.guest if user.is_guest else Role.user
+    data["autocreated"] = True
+
+    _ensure_auth_user(user_data=data, company_id=user.company, log=log)
+
     db_user = User.objects(company=user.company, id=user.user_id).first()
     if db_user:
         # noinspection PyBroadException
         try:
             log.info(f"Updating user name: {user.name}")
             given_name, _, family_name = user.name.partition(" ")
-            db_user.update(name=user.name, given_name=given_name, family_name=family_name)
+            db_user.update(
+                name=user.name, given_name=given_name, family_name=family_name
+            )
         except Exception:
             pass
-        return
+    else:
+        _ensure_backend_user(user.user_id, user.company, user.name)
 
-    data = attr.asdict(user)
-    data["id"] = user.user_id
-    data["email"] = f"{user.user_id}@example.com"
-    data["role"] = Role.guest if user.is_guest else Role.user
-
-    _ensure_auth_user(user_data=data, company_id=user.company, log=log)
-
-    return _ensure_backend_user(user.user_id, user.company, user.name)
+    emails.add(email)

apiserver/requirements.txt

@@ -6,11 +6,11 @@ boto3>=1.26
 boto3-stubs[s3]>=1.26
 clearml>=1.10.3
 dpath>=1.4.2,<2.0
-elasticsearch==7.17.9
+elasticsearch==8.12.0
 fastjsonschema>=2.8
 flask-compress>=1.4.0
 flask-cors>=3.0.5
-flask>=2.3.2
+flask>=2.3.3
 furl>=2.0.0
 google-cloud-storage>=2.8.0
 gunicorn>=20.1.0
@@ -25,7 +25,7 @@ packaging==20.3
 psutil>=5.6.5
 pyhocon>=0.3.35r
 pyjwt>=2.4.0
-pymongo==4.4.0
+pymongo==4.7.3
 python-rapidjson>=0.6.3
 redis>=4.5.4,<5
 requests>=2.13.0
@@ -33,4 +33,5 @@ semantic_version>=2.8.3,<3
 setuptools>=65.5.1
 six
 validators>=0.12.4
-urllib3>=1.26.16
+urllib3>=1.26.18
+werkzeug>=3.0.1

apiserver/schema/services/_common.conf

@@ -1,3 +1,43 @@
+field_filter {
+    type: object
+    description: Filter on a field that includes combination of 'any' or 'all' included and excluded terms
+    properties {
+        any {
+            type: object
+            description: All the terms in 'any' condition are combined with 'or' operation
+            properties {
+                "include" {
+                    type: array
+                    items {type: string}
+                }
+                exclude {
+                    type: array
+                    items {type: string}
+                }
+            }
+        }
+        all {
+            type: object
+            description: All the terms in 'all' condition are combined with 'and' operation
+            properties {
+                "include" {
+                    type: array
+                    items {type: string}
+                }
+                exclude {
+                    type: array
+                    items {type: string}
+                }
+            }
+        }
+        op {
+            type: string
+            description: The operation between 'any' and 'all' parts of the filter if both are provided
+            default: and
+            enum: [and, or]
+        }
+    }
+}
 metadata_item {
     type: object
     properties {
@@ -34,7 +74,11 @@ multi_field_pattern_data {
     type: object
     properties {
         pattern {
-            description: "Pattern string (regex)"
+            description: "Pattern string (regex). Either 'pattern' or 'datetime' should be specified"
+            type: string
+        }
+        datetime {
+            description: "Date time conditions (applicable only to datetime fields). Either 'pattern' or 'datetime' should be specified"
             type: string
         }
         fields {

apiserver/schema/services/_tasks_common.conf

@@ -283,6 +283,22 @@ last_metrics_event {
             description: "The iteration at which the maximum value was reported"
             type: integer
         }
+        first_value {
+            description: "First value reported"
+            type: number
+        }
+        first_value_iteration {
+            description: "The iteration at which the first value was reported"
+            type: integer
+        }
+        mean_value {
+            description: "The mean value"
+            type: number
+        }
+        count {
+            description: "The total count of reported values"
+            type: integer
+        }
     }
 }
 last_metrics_variants {
@@ -414,7 +430,7 @@ task {
         container {
             description: "Docker container parameters"
             type: object
-            additionalProperties { type: [string, null] }
+            additionalProperties { type: string }
         }
         models {
             description: "Task models"

apiserver/schema/services/_workers_common.conf

@@ -0,0 +1,67 @@
+machine_stats  {
+    type: object
+    properties {
+        cpu_usage {
+            description: "Average CPU usage per core"
+            type: array
+            items { type: number }
+        }
+        gpu_usage {
+            description: "Average GPU usage per GPU card"
+            type: array
+            items { type: number }
+        }
+        memory_used {
+            description: "Used memory MBs"
+            type: number
+        }
+        memory_free {
+            description: "Free memory MBs"
+            type: number
+        }
+        gpu_memory_free {
+            description: "GPU free memory MBs"
+            type: array
+            items { type: number }
+        }
+        gpu_memory_used {
+            description: "GPU used memory MBs"
+            type: array
+            items { type: number }
+        }
+        network_tx {
+            description: "Mbytes per second"
+            type: number
+        }
+        network_rx {
+            description: "Mbytes per second"
+            type: number
+        }
+        disk_free_home {
+            description: "Free space in % of /home drive"
+            type: number
+        }
+        disk_free_temp {
+            description: "Free space in % of /tmp drive"
+            type: number
+        }
+        disk_read {
+            description: "Mbytes read per second"
+            type: number
+        }
+        disk_write {
+            description: "Mbytes write per second"
+            type: number
+        }
+        cpu_temperature {
+            description: "CPU temperature"
+            type: array
+            items { type: number }
+        }
+        gpu_temperature {
+            description: "GPU temperature"
+            type: array
+            items { type: number }
+        }
+    }
+}

apiserver/schema/services/events.conf

@@ -11,8 +11,8 @@ _definitions {
                 type: number
             }
             type {
-                description: "training_stats_vector"
-                const: "training_stats_scalar"
+                description: "'training_stats_scalar'"
+                type: string
             }
             task {
                 description: "Task ID (required)"
@@ -46,8 +46,8 @@ _definitions {
                 type: number
             }
             type {
-                description: "training_stats_vector"
-                const: "training_stats_vector"
+                description: "'training_stats_vector'"
+                type: string
             }
             task {
                 description: "Task ID (required)"
@@ -82,8 +82,8 @@ _definitions {
                 type: number
             }
             type {
-                description: ""
-                const: "training_debug_image"
+                description: "'training_debug_image'"
+                type: string
             }
             task {
                 description: "Task ID (required)"
@@ -123,7 +123,7 @@ _definitions {
             }
             type {
                 description: "'plot'"
-                const: "plot"
+                type: string
             }
             task {
                 description: "Task ID (required)"
@@ -221,7 +221,7 @@ _definitions {
             }
             type {
                 description: "'log'"
-                const: "log"
+                type: string
             }
             task {
                 description: "Task ID (required)"
@@ -754,6 +754,42 @@ get_task_metrics{
         }
     }
 }
+get_multi_task_metrics {
+    "2.28" {
+        description: """Get unique metrics and variants from the events of the specified type.
+        Only events reported for the passed task or model ids are analyzed."""
+        request {
+            type: object
+            required: [ tasks ]
+            properties {
+                tasks {
+                    description: task ids to get metrics from
+                    type: array
+                    items {type: string}
+                }
+                model_events {
+                    description: If not set or set to false then passed ids are task ids otherwise model ids
+                    type: boolean
+                    default: false
+                }
+                event_type {
+                    "description": Event type. If not specified then metrics are collected from the reported events of all types
+                    "$ref": "#/definitions/event_type_enum"
+                }
+            }
+        }
+        response {
+            type: object
+            properties {
+                metrics {
+                    type: array
+                    description: List of metrics and variants
+                    items { "$ref": "#/definitions/metric_variants" }
+                }
+            }
+        }
+    }
+}
 get_task_log {
     "1.5" {
         description: "Get all 'log' events for this task"
@@ -911,6 +947,13 @@ get_task_log {
             }
         }
     }
+    "2.30": ${get_task_log."2.9"} {
+        request.metrics {
+            type: array
+            description: List of metrics and variants
+            items { "$ref": "#/definitions/metric_variants" }
+        }
+    }
 }
 get_task_events {
     "2.1" {
@@ -971,10 +1014,17 @@ get_task_events {
         }
     }
     "2.22": ${get_task_events."2.1"} {
-        request.properties.model_events {
-            type: boolean
-            description: If set then get retrieving model events. Otherwise task events
-            default: false
+        request.properties {
+            model_events {
+                type: boolean
+                description: If set then get retrieving model events. Otherwise task events
+                default: false
+            }
+            metrics {
+                type: array
+                description: List of metrics and variants
+                items { "$ref": "#/definitions/metric_variants" }
+            }
         }
     }
 }
@@ -1156,6 +1206,13 @@ get_multi_task_plots {
             default: true
         }
     }
+    "2.28": ${get_multi_task_plots."2.26"} {
+        request.properties.metrics {
+            type: array
+            description: List of metrics and variants
+            items { "$ref": "#/definitions/metric_variants" }
+        }
+    }
 }
 get_vector_metrics_and_variants {
     "2.1" {
@@ -1342,6 +1399,13 @@ multi_task_scalar_metrics_iter_histogram {
             default: false
         }
     }
+    "2.28": ${multi_task_scalar_metrics_iter_histogram."2.22"} {
+        request.properties.metrics {
+            type: array
+            description: List of metrics and variants
+            items { "$ref": "#/definitions/metric_variants" }
+        }
+    }
 }
 get_task_single_value_metrics {
     "2.20" {
@@ -1369,6 +1433,13 @@ get_task_single_value_metrics {
             default: false
         }
     }
+    "2.28": ${get_task_single_value_metrics."2.22"} {
+        request.properties.metrics {
+            type: array
+            description: List of metrics and variants
+            items { "$ref": "#/definitions/metric_variants" }
+        }
+    }
 }
 get_task_latest_scalar_values {
     "2.1" {
@@ -1470,6 +1541,10 @@ get_scalar_metric_data {
                     type: string
                     description: type of metric
                 }
+                scroll_id {
+                    type: string
+                    description: "Scroll ID of previous call (used for getting more results)"
+                }
             }
         }
         response {
@@ -1492,7 +1567,7 @@ get_scalar_metric_data {
                 }
                 scroll_id {
                     type: string
-                    description: "Scroll ID of previous call (used for getting more results)"
+                    description: "Scroll ID for getting more results"
                 }
             }
         }
@@ -1637,4 +1712,18 @@ clear_task_log {
             }
         }
     }
+    "2.30": ${clear_task_log."2.19"} {
+        request.properties {
+            include_metrics {
+                type: array
+                description: If passed then only events for these metrics are deleted
+                items: {type: string}
+            }
+            exclude_metrics {
+                type: array
+                description: If passed then events for these metrics are retained
+                items: {type: string}
+            }
+        }
+    }
 }

apiserver/schema/services/login.conf

@@ -6,21 +6,12 @@ _default {
 }
 
 supported_modes {
-    authorize: false
+    authorize: null
     "2.9" {
         description:  """ Return supported login modes."""
         request {
             type: object
-            properties {
-                state {
-                    description: "ASCII base64 encoded application state"
-                    type: string
-                }
-                callback_url_prefix {
-                    description: "URL prefix used to generate the callback URL for each supported SSO provider"
-                    type: string
-                }
-            }
+            additionalProperties: false
         }
         response {
             type: object
@@ -59,7 +50,7 @@ supported_modes {
                     description: "SSO authentication providers"
                     type: object
                     additionalProperties {
-                        desctiprion: "Provider redirect URL"
+                        description: "Provider redirect URL"
                         type: string
                     }
                 }
@@ -95,7 +86,7 @@ supported_modes {
 }
 
 logout {
-    authorize: false
+    authorize: null
     allow_roles = [ "*" ]
     "2.13" {
         description: """ Logout (including SSO, if used)) """

apiserver/schema/services/models.conf

@@ -1,20 +1,6 @@
 _description: """This service provides a management interface for models (results of training tasks) stored in the system."""
 _definitions {
     include "_tasks_common.conf"
-    multi_field_pattern_data {
-        type: object
-        properties {
-            pattern {
-                description: "Pattern string (regex)"
-                type: string
-            }
-            fields {
-                description: "List of field names"
-                type: array
-                items { type: string }
-            }
-        }
-    }
     model {
         type: object
         properties {
@@ -261,6 +247,14 @@ get_all_ex {
             }
         }
     }
+    "2.27": ${get_all_ex."2.23"} {
+        request.properties {
+            filters {
+                type: object
+                additionalProperties: ${_definitions.field_filter}
+            }
+        }
+    }
 }
 get_all {
     "2.1" {
@@ -341,7 +335,7 @@ get_all {
                     items { type: string }
                 }
                 last_update {
-                    description: "List of last_update constraint strings (utcformat, epoch) with an optional prefix modifier (>, >=, <, <=)"
+                    description: "List of last_update constraint strings (utcformat, epoch) with an optional prefix modifier (\>, \>=, \<, \<=)"
                     type: array
                     items {
                         type: string
@@ -357,9 +351,6 @@ get_all {
                     "$ref": "#/definitions/multi_field_pattern_data"
                 }
             }
-            dependencies {
-                page: [ page_size ]
-            }
         }
         response {
             type: object
@@ -1086,4 +1077,38 @@ delete_metadata {
             }
         }
     }
+}
+update_tags {
+    "2.27" {
+        description: Add or remove tags from multiple models
+        request {
+            type: object
+            properties {
+                ids {
+                    type: array
+                    description: IDs of the models to update
+                    items {type: string}
+                }
+                add_tags {
+                    type: array
+                    description: User tags to add
+                    items {type: string}
+                }
+                remove_tags {
+                    type: array
+                    description: User tags to remove
+                    items {type: string}
+                }
+            }
+        }
+        response {
+            type: object
+            properties {
+                updated {
+                    type: integer
+                    description: The number of updated models
+                }
+            }
+        }
+    }
 }

apiserver/schema/services/organization.conf

@@ -203,7 +203,7 @@ get_entities_count {
                 default: false
             }
             active_users {
-                descritpion: "The list of users that were active in the project. If passes then the resulting projects are filtered to the ones that have tasks created by these users"
+                description: "The list of users that were active in the project. If passes then the resulting projects are filtered to the ones that have tasks created by these users"
                 type: array
                 items: {type: string}
             }

apiserver/schema/services/pipelines.conf

@@ -59,7 +59,7 @@ start_pipeline {
                         type: object
                         properties {
                             name: { type: string }
-                            value: { type: [string, null] }
+                            value: { type: string }
                         }
                     }
                 }
@@ -79,4 +79,15 @@ start_pipeline {
             }
         }
     }
+    "2.28": ${start_pipeline."2.17"} {
+        request.properties.verify_watched_queue {
+            description: If passed then check wheter there are any workers watiching the queue
+            type: boolean
+            default: false
+        }
+        response.properties.queue_watched {
+            description: Returns true if there are workers or autscalers working with the queue
+            type: boolean
+        }
+    }
 }

apiserver/schema/services/projects.conf

@@ -1,19 +1,6 @@
 _description: "Provides support for defining Projects containing Tasks, Models and Dataset Versions."
 _definitions {
-    multi_field_pattern_data {
-        type: object
-        properties {
-            pattern {
-                description: "Pattern string (regex)"
-                type: string
-            }
-            fields {
-                description: "List of field names"
-                type: array
-                items { type: string }
-            }
-        }
-    }
+    include "_common.conf"
     project {
         type: object
         properties {
@@ -569,7 +556,7 @@ get_all_ex {
         request {
             properties {
                 active_users {
-                    descritpion: "The list of users that were active in the project. If passes then the resulting projects are filtered to the ones that have tasks created by these users"
+                    description: "The list of users that were active in the project. If passes then the resulting projects are filtered to the ones that have tasks created by these users"
                     type: array
                     items: {type: string}
                 }
@@ -660,6 +647,15 @@ get_all_ex {
             items {type: string}
         }
     }
+    "2.27": ${get_all_ex."2.25"} {
+        request.properties {
+            filters {
+                type: object
+                additionalProperties: ${_definitions.field_filter}
+            }
+            children_tags_filter: ${_definitions.field_filter}
+        }
+    }
 }
 update {
     "2.1" {
@@ -939,6 +935,13 @@ get_unique_metric_variants {
             default: false
         }
     }
+    "2.28": ${get_unique_metric_variants."2.25"} {
+        request.properties.ids {
+            description: IDs of the tasks or models to get metrics from
+            type: array
+            items {type: string}
+        }
+    }
 }
 get_hyperparam_values {
     "2.13" {
@@ -1000,6 +1003,12 @@ get_hyperparam_values {
             }
         }
     }
+    "2.27": ${get_hyperparam_values."2.26"} {
+        request.properties.pattern {
+            type: string
+            description: The search pattern regex
+        }
+    }
 }
 get_hyper_parameters {
     "2.9" {
@@ -1270,13 +1279,15 @@ get_task_parents {
                             }
                             project {
                                 type: object
-                                id {
-                                    description: "The ID of the parent task project"
-                                    type: string
-                                }
-                                name {
-                                    description: "The name of the parent task project"
-                                    type: string
+                                properties {
+                                    id {
+                                        description: "The ID of the parent task project"
+                                        type: string
+                                    }
+                                    name {
+                                        description: "The name of the parent task project"
+                                        type: string
+                                    }
                                 }
                             }
                         }

apiserver/schema/services/queues.conf

@@ -159,6 +159,14 @@ get_all_ex {
             default: false
         }
     }
+    "2.27": ${get_all_ex."2.21"} {
+        request.properties {
+            filters {
+                type: object
+                additionalProperties: ${_definitions.field_filter}
+            }
+        }
+    }
 }
 get_all {
     "2.4" {
@@ -439,6 +447,13 @@ add_task {
             }
         }
     }
+    "2.31": ${add_task."2.4"} {
+        request.properties.update_execution_queue {
+            description: If set to false then the task 'execution.queue' is not updated
+            type: boolean
+            default: true
+        }
+    }
 }
 get_next_task {
     "2.4" {
@@ -522,8 +537,41 @@ remove_task {
             }
         }
     }
+    "2.31": ${remove_task."2.4"} {
+        request.properties {
+            update_task_status {
+                type: boolean
+                default: false
+                description: If set to 'true' then change the removed task status to the one it had prior to enqueuing or 'created'
+            }
+        }
+    }
+}
+clear_queue {
+    "2.31" {
+        description: Remove all tasks from the queue and change their statuses to what they were prior to enqueuing or 'created'
+        request {
+            type: object
+            required: [queue]
+            properties {
+                queue {
+                    description: "Queue id"
+                    type: string
+                }
+            }
+        }
+        response {
+            type: object
+            properties {
+                removed_tasks {
+                    description: IDs of the removed tasks
+                    type: array
+                    items {type: string}
+                }
+            }
+        }
+    }
 }
-
 move_task_forward: {
     "2.4" {
         description: "Moves a task entry one step forward towards the top of the queue."

apiserver/schema/services/reports.conf

@@ -446,7 +446,7 @@ get_task_data {
                     type: string
                 }
                 status_changed {
-                    description: "List of status changed constraint strings (utcformat, epoch) with an optional prefix modifier (>, >=, <, <=)"
+                    description: "List of status changed constraint strings (utcformat, epoch) with an optional prefix modifier (\>, \>=, \<, \<=)"
                     type: array
                     items {
                         type: string
@@ -578,7 +578,7 @@ get_task_data {
             single_value_metrics {
                 type: object
                 description: If passed then task single value metrics are returned
-                additonalProperties: false
+                additionalProperties: false
             }
         }
         response.properties.single_value_metrics {
@@ -656,7 +656,7 @@ get_all_ex {
                     items { type: string }
                 }
                 status_changed {
-                    description: "List of status changed constraint strings (utcformat, epoch) with an optional prefix modifier (>, >=, <, <=)"
+                    description: "List of status changed constraint strings (utcformat, epoch) with an optional prefix modifier (\>, \>=, \<, \<=)"
                     type: array
                     items {
                         type: string
@@ -694,9 +694,6 @@ get_all_ex {
                     "$ref": "#/definitions/multi_field_pattern_data"
                 }
             }
-            dependencies {
-                page: [ page_size ]
-            }
         }
         response {
             type: object
@@ -720,6 +717,14 @@ get_all_ex {
             default: false
         }
     }
+    "2.27": ${get_all_ex."2.26"} {
+        request.properties {
+            filters {
+                type: object
+                additionalProperties: ${_definitions.field_filter}
+            }
+        }
+    }
 }
 get_tags {
     "2.23" {

apiserver/schema/services/serving.conf

@@ -0,0 +1,437 @@
+_description: "Serving apis"
+_definitions {
+    include "_workers_common.conf"
+    reference_item {
+        type: object
+        required = [type, value]
+        properties {
+            type {
+                description: The type of the reference item
+                type: string
+                enum: [app_id, app_instance, model, task, url]
+            }
+            value {
+                description: The reference item value
+                type: string
+            }
+        }
+    }
+    reference {
+        description: Array of reference items provided by the container instance. Can contain multiple reference items with the same type
+        type: array
+        items: ${_definitions.reference_item}
+    }
+    serving_model_report {
+        type: object
+        required: [container_id, endpoint_name, model_name]
+        properties {
+            container_id {
+                type: string
+                description: Container ID. Should uniquely identify a specific container instance
+            }
+            endpoint_name {
+                type: string
+                description: Endpoint name
+            }
+            endpoint_url {
+                type: string
+                description: Endpoint URL
+            }
+            model_name {
+                type: string
+                description: Model name
+            }
+            model_source {
+                type: string
+                description: Model source
+            }
+            model_version {
+                type: string
+                description: Model version
+            }
+            preprocess_artifact {
+                type: string
+                description: Preprocess Artifact
+            }
+            input_type {
+                type: string
+                description: Input type
+            }
+            input_size {
+                type: string
+                description: Input size
+            }
+            reference: ${_definitions.reference}
+        }
+    }
+    endpoint_stats {
+        type: object
+        properties {
+            endpoint {
+                type: string
+                description: Endpoint name
+            }
+            model {
+                type: string
+                description: Model name
+            }
+            url {
+                type: string
+                description: Model url
+            }
+            instances {
+                type: integer
+                description: The number of model serving instances
+            }
+            uptime_sec {
+                type: integer
+                description: Max of model instance uptime in seconds
+            }
+            requests {
+                type: integer
+                description: Total requests processed by model instances
+            }
+            requests_min {
+                type: number
+                description: Average of request rate of model instances per minute
+            }
+            latency_ms {
+                type: integer
+                description: Average of latency of model instances in ms
+            }
+            last_update {
+                type: string
+                format: "date-time"
+                description: The latest time when one of the model instances was updated
+            }
+        }
+    }
+    container_instance_stats {
+        type: object
+        properties {
+            id {
+                type: string
+                description: Container ID
+            }
+            uptime_sec {
+                type: integer
+                description: Uptime in seconds
+            }
+            requests {
+                type: integer
+                description: Number of requests
+            }
+            requests_min {
+                type: number
+                description: Average requests per minute
+            }
+            latency_ms {
+                type: integer
+                description: Average request latency in ms
+            }
+            last_update {
+                type: string
+                format: "date-time"
+                description: The latest time when the container instance sent update
+            }
+            cpu_count {
+                type: integer
+                description: CPU Count
+            }
+            gpu_count {
+                type: integer
+                description: GPU Count
+            }
+            reference: ${_definitions.reference}
+
+        }
+    }
+    serving_model_info {
+        type: object
+        properties {
+            endpoint {
+                type: string
+                description: Endpoint name
+            }
+            model {
+                type: string
+                description: Model name
+            }
+            url {
+                type: string
+                description: Model url
+            }
+            model_source {
+                type: string
+                description: Model source
+            }
+            model_version {
+                type: string
+                description: Model version
+            }
+            preprocess_artifact {
+                type: string
+                description: Preprocess Artifact
+            }
+            input_type {
+                type: string
+                description: Input type
+            }
+            input_size {
+                type: string
+                description: Input size
+            }
+        }
+    }
+    container_info: ${_definitions.serving_model_info} {
+        properties {
+            id {
+                type: string
+                description: Container ID
+            }
+            uptime_sec {
+                type: integer
+                description: Model instance uptime in seconds
+            }
+            last_update {
+                type: string
+                format: "date-time"
+                description: The latest time when the container instance sent update
+            }
+            age_sec {
+                type: integer
+                description: Amount of seconds since the container registration
+            }
+        }
+    }
+    metrics_history_series {
+        type: object
+        properties {
+            title {
+                type: string
+                description: "The title of the series"
+            }
+            dates {
+                type: array
+                description: "List of timestamps (in seconds from epoch) in the acceding order. The timestamps are separated by the requested interval."
+                items {type: integer}
+            }
+            values {
+                type: array
+                description: "List of values corresponding to the timestamps in the dates list."
+                items {type: number}
+            }
+        }
+    }
+}
+register_container {
+    "2.31" {
+        description: Register container
+        request = ${_definitions.serving_model_report} {
+            properties {
+                timeout {
+                    description: "Registration timeout in seconds. If timeout seconds have passed since the service container last call to register or status_report, the container is automatically removed from the list of registered containers."
+                    type: integer
+                    default: 600
+                }
+            }
+        }
+        response {
+            type: object
+            additionalProperties: false
+        }
+    }
+}
+unregister_container {
+    "2.31" {
+        description: Unregister container
+        request {
+            type: object
+            required: [container_id]
+            properties {
+                container_id {
+                    type: string
+                    description: Container ID
+                }
+            }
+        }
+        response {
+            type: object
+            additionalProperties: false
+        }
+    }
+}
+container_status_report {
+    "2.31" {
+        description: Container status report
+        request = ${_definitions.serving_model_report} {
+            properties {
+                uptime_sec {
+                    type: integer
+                    description: Uptime in seconds
+                }
+                requests_num {
+                    type: integer
+                    description: Number of requests
+                }
+                requests_min {
+                    type: number
+                    description: Average requests per minute
+                }
+                latency_ms {
+                    type: integer
+                    description: Average request latency in ms
+                }
+                machine_stats {
+                    description: "The machine statistics"
+                    "$ref": "#/definitions/machine_stats"
+                }
+            }
+        }
+        response {
+            type: object
+            additionalProperties: false
+        }
+    }
+}
+get_endpoints {
+    "2.31" {
+        description: Get all the registered endpoints
+        request {
+            type: object
+            additionalProperties: false
+        }
+        response {
+            type: object
+            properties {
+                endpoints {
+                    type: array
+                    items { "$ref": "#/definitions/endpoint_stats" }
+                }
+            }
+        }
+    }
+}
+get_loading_instances {
+    "2.31" {
+        description: "Get loading instances (enpoint_url not set yet)"
+        request {
+            type: object
+            additionalProperties: false
+        }
+        response {
+            type: object
+            properties {
+                instances {
+                    type: array
+                    items { "$ref": "#/definitions/container_info" }
+                }
+            }
+        }
+    }
+}
+get_endpoint_details {
+    "2.31" {
+        description: Get endpoint details
+        request {
+            type: object
+            required: [endpoint_url]
+            properties {
+                endpoint_url {
+                    type: string
+                    description: Endpoint URL
+                }
+            }
+        }
+        response: ${_definitions.serving_model_info} {
+            properties {
+                uptime_sec {
+                    type: integer
+                    description: Max of model instance uptime in seconds
+                }
+                last_update {
+                    type: string
+                    format: "date-time"
+                    description: The latest time when one of the model instances was updated
+                }
+                instances {
+                    type: array
+                    items {"$ref": "#/definitions/container_instance_stats"}
+                }
+            }
+        }
+    }
+}
+get_endpoint_metrics_history {
+    "2.31" {
+        description: Get endpoint charts
+        request {
+            type: object
+            required: [endpoint_url, from_date, to_date, interval]
+            properties {
+                endpoint_url {
+                    description: Endpoint Url
+                    type: string
+                }
+                from_date {
+                    description: "Starting time (in seconds from epoch) for collecting statistics"
+                    type: number
+                }
+                to_date {
+                    description: "Ending time (in seconds from epoch) for collecting statistics"
+                    type: number
+                }
+                interval {
+                    description: "Time interval in seconds for a single statistics point. The minimal value is 1"
+                    type: integer
+                }
+                metric_type {
+                    description: The type of the metrics to return on the chart
+                    type: string
+                    default: requests
+                    enum: [
+                        requests
+                        requests_min
+                        latency_ms
+                        cpu_count
+                        gpu_count
+                        cpu_util
+                        gpu_util
+                        ram_total
+                        ram_used
+                        ram_free
+                        gpu_ram_total
+                        gpu_ram_used
+                        gpu_ram_free
+                        network_rx
+                        network_tx
+                    ]
+                }
+                instance_charts {
+                    type: boolean
+                    default: true
+                    description: If set then return instance charts and total. Otherwise total only
+                }
+            }
+        }
+        response {
+            type: object
+            properties {
+                computed_interval {
+                    description: The inteval that was actually used for the histogram. May be larger then the requested one
+                    type: integer
+                }
+                total: ${_definitions.metrics_history_series} {
+                    properties {
+                        description: The total histogram
+                    }
+                }
+                instances {
+                    description: Instance charts
+                    type: object
+                    additionalProperties: ${_definitions.metrics_history_series}
+                }
+            }
+        }
+    }
+}

apiserver/schema/services/storage.conf

@@ -0,0 +1,242 @@
+_description: """This service provides storage settings managmement"""
+_default {
+    internal: true
+}
+
+_definitions {
+    include "_common.conf"
+    aws_bucket {
+        type: object
+        description: Settings per S3 bucket
+        properties {
+            bucket {
+                description: The name of the bucket
+                type: string
+            }
+            subdir {
+                description: The path to match
+                type: string
+            }
+            host {
+                description: Host address (for minio servers)
+                type: string
+            }
+            key {
+                description: Access key
+                type: string
+            }
+            secret {
+                description: Secret key
+                type: string
+            }
+            token {
+                description: Access token
+                type: string
+            }
+            multipart {
+                description: Multipart upload
+                type: boolean
+                default: true
+            }
+            acl {
+                description: ACL
+                type: string
+            }
+            secure {
+                description: Use SSL connection
+                type: boolean
+                default: true
+            }
+            region {
+                description: AWS Region
+                type: string
+            }
+            verify {
+                description: Verify server certificate
+                type: boolean
+                default: true
+            }
+            use_credentials_chain {
+                description: Use host configured credentials
+                type: boolean
+                default: false
+            }
+        }
+    }
+    aws {
+        type: object
+        description: AWS S3 storage settings
+        properties {
+            key {
+                description: Access key
+                type: string
+            }
+            secret {
+                description: Secret key
+                type: string
+            }
+            region {
+                description: AWS region
+                type: string
+            }
+            token {
+                description: Access token
+                type: string
+            }
+            use_credentials_chain {
+                description: If set then use host credentials
+                type: boolean
+                default: false
+            }
+            buckets {
+                description: Credential settings per bucket
+                type: array
+                items {"$ref": "#/definitions/aws_bucket"}
+            }
+        }
+    }
+    google_bucket {
+        type: object
+        description: Settings per Google storage bucket
+        properties {
+            bucket {
+                description: The name of the bucket
+                type: string
+            }
+            project {
+                description: The name of the project
+                type: string
+            }
+            subdir {
+                description: The path to match
+                type: string
+            }
+            credentials_json {
+                description: The contents of the credentials json file
+                type: string
+            }
+        }
+    }
+    google {
+        type: object
+        description: Google storage settings
+        properties {
+            project {
+                description: Project name
+                type: string
+            }
+            credentials_json {
+                description: The contents of the credentials json file
+                type: string
+            }
+            buckets {
+                description: Credentials per bucket
+                type: array
+                items {"$ref": "#/definitions/google_bucket"}
+            }
+        }
+    }
+    azure_container {
+        type: object
+        description: Azure container settings
+        properties {
+            account_name {
+                description: Account name
+                type: string
+            }
+            account_key {
+                description: Account key
+                type: string
+            }
+            container_name {
+                description: The name of the container
+                type: string
+            }
+        }
+    }
+    azure {
+        type: object
+        description: Azure storage settings
+        properties {
+            containers {
+                description: Credentials per container
+                type: array
+                items {"$ref": "#/definitions/azure_container"}
+            }
+        }
+    }
+}
+
+set_settings {
+    "2.31" {
+        description: Set Storage settings
+        request {
+            type: object
+            properties {
+                aws {"$ref": "#/definitions/aws"}
+                google {"$ref": "#/definitions/google"}
+                azure {"$ref": "#/definitions/azure"}
+            }
+        }
+        response {
+            type: object
+            properties {
+                updated {
+                    description: "Number of settings documents updated (0 or 1)"
+                    type: integer
+                    enum: [0, 1]
+                }
+            }
+        }
+    }
+}
+reset_settings {
+    "2.31" {
+        description: Reset selected storage settings
+        request {
+            type: object
+            properties {
+                keys {
+                    description: The names of the settings to delete
+                    type: array
+                    items {
+                        type: string
+                        enum: ["azure", "aws", "google"]
+                    }
+                }
+            }
+        }
+        response {
+            type: object
+            properties {
+                updated {
+                    description: "Number of settings documents updated (0 or 1)"
+                    type: integer
+                    enum: [0, 1]
+                }
+            }
+        }
+    }
+}
+get_settings {
+    "2.22" {
+        description: Get storage settings
+        request {
+            type: object
+            additionalProperties: false
+        }
+        response {
+            type: object
+            properties {
+                last_update {
+                    description: "Settings last update time (UTC) "
+                    type: string
+                    format: "date-time"
+                }
+                aws {"$ref": "#/definitions/aws"}
+                google {"$ref": "#/definitions/google"}
+                azure {"$ref": "#/definitions/azure"}
+            }
+        }
+    }
+}

apiserver/schema/services/tasks.conf

@@ -190,6 +190,14 @@ get_all_ex {
             }
         }
     }
+    "2.27": ${get_all_ex."2.23"} {
+        request.properties {
+            filters {
+                type: object
+                additionalProperties: ${_definitions.field_filter}
+            }
+        }
+    }
 }
 get_all {
     "2.1" {
@@ -269,7 +277,7 @@ get_all {
                     type: string
                 }
                 status_changed {
-                    description: "List of status changed constraint strings (utcformat, epoch) with an optional prefix modifier (>, >=, <, <=)"
+                    description: "List of status changed constraint strings (utcformat, epoch) with an optional prefix modifier (\>, \>=, \<, \<=)"
                     type: array
                     items {
                         type: string
@@ -289,9 +297,6 @@ get_all {
                     "$ref": "#/definitions/multi_field_pattern_data"
                 }
             }
-            dependencies {
-                page: [ page_size ]
-            }
         }
         response {
             type: object
@@ -481,7 +486,7 @@ clone {
                 new_task_container {
                     description: "The docker container properties for the new task. If not provided then taken from the original task"
                     type: object
-                    additionalProperties { type: [string, null] }
+                    additionalProperties { type: string }
                 }
             }
         }
@@ -659,7 +664,7 @@ create {
                 container {
                     description: "Docker container parameters"
                     type: object
-                    additionalProperties { type: [string, null] }
+                    additionalProperties { type: string }
                 }
             }
         }
@@ -748,7 +753,7 @@ validate {
                 container {
                     description: "Docker container parameters"
                     type: object
-                    additionalProperties { type: [string, null] }
+                    additionalProperties { type: string }
                 }
             }
         }
@@ -910,7 +915,7 @@ edit {
                 container {
                     description: "Docker container parameters"
                     type: object
-                    additionalProperties { type: [string, null] }
+                    additionalProperties { type: string }
                 }
                 runtime {
                     description: "Task runtime mapping"
@@ -1102,6 +1107,13 @@ delete_many {
             default: true
         }
     }
+    "2.30": ${delete_many."2.21"} {
+        request.properties.include_pipeline_steps {
+            description: If set then for the passed pipeline controller tasks the pipeline steps will be also deleted
+            type: boolean
+            default: false
+        }
+    }
 }
 delete {
     "2.1" {
@@ -1177,6 +1189,13 @@ delete {
             default: true
         }
     }
+    "2.30": ${delete."2.21"} {
+        request.properties.include_pipeline_steps {
+            description: If set then and the passed task is a pipeline controller then delete the pipeline tasks too
+            type: boolean
+            default: false
+        }
+    }
 }
 archive {
     "2.12" {
@@ -1214,6 +1233,13 @@ archive {
             }
         }
     }
+    "2.30": ${archive."2.12"} {
+        request.properties.include_pipeline_steps {
+            description: If set then for the passed pipeline controller tasks also archive the pipeline steps
+            type: boolean
+            default: false
+        }
+    }
 }
 archive_many {
     "2.13": ${_definitions.batch_operation} {
@@ -1240,6 +1266,13 @@ archive_many {
             }
         }
     }
+    "2.30": ${archive_many."2.13"} {
+        request.properties.include_pipeline_steps {
+            description: If set then for the passed pipeline controller tasks also archive the pipeline steps
+            type: boolean
+            default: false
+        }
+    }
 }
 unarchive_many {
     "2.13": ${_definitions.batch_operation} {
@@ -1266,6 +1299,13 @@ unarchive_many {
             }
         }
     }
+    "2.30": ${unarchive_many."2.13"} {
+        request.properties.include_pipeline_steps {
+            description: If set then for the passed pipeline controller tasks also archive the pipeline steps
+            type: boolean
+            default: false
+        }
+    }
 }
 started {
     "2.1" {
@@ -1304,6 +1344,13 @@ stop {
         } ${_references.status_change_request}
         response: ${_definitions.update_response}
     }
+    "2.30": ${stop."2.1"} {
+        request.properties.include_pipeline_steps {
+            description: If set and the passed task is a pipeline controller then stop all its steps too
+            type: boolean
+            default: false
+        }
+    }
 }
 stop_many {
     "2.13": ${_definitions.change_many_request} {
@@ -1317,6 +1364,13 @@ stop_many {
             }
         }
     }
+    "2.30": ${stop_many."2.13"} {
+        request.properties.include_pipeline_steps {
+            description: If set then for all the passed pipeline controller tasks stop their steps too
+            type: boolean
+            default: false
+        }
+    }
 }
 stopped {
     "2.1" {
@@ -1453,6 +1507,13 @@ Fails if the following parameters in the task were not filled:
             type: boolean
         }
     }
+    "2.31": ${enqueue."2.22"} {
+        request.properties.update_execution_queue {
+            description: If set to false then the task 'execution.queue' is not updated. This can be done only for the task that is already enqueued
+            type: boolean
+            default: true
+        }
+    }
 }
 enqueue_many {
     "2.13": ${_definitions.change_many_request} {
@@ -2050,3 +2111,37 @@ move {
         }
     }
 }
+update_tags {
+    "2.27" {
+        description: Add or remove tags from multiple tasks
+        request {
+            type: object
+            properties {
+                ids {
+                    type: array
+                    description: IDs of the tasks to update
+                    items {type: string}
+                }
+                add_tags {
+                    type: array
+                    description: User tags to add
+                    items {type: string}
+                }
+                remove_tags {
+                    type: array
+                    description: User tags to remove
+                    items {type: string}
+                }
+            }
+        }
+        response {
+            type: object
+            properties {
+                updated {
+                    type: integer
+                    description: The number of updated tasks
+                }
+            }
+        }
+    }
+}