From c7cd949fd088649f91dd61fa73426f11840c0e4d Mon Sep 17 00:00:00 2001 From: allegroai <> Date: Wed, 21 Dec 2022 18:30:54 +0200 Subject: [PATCH] Add reports support Fix schema --- apiserver/apierrors/errors.conf | 3 + apiserver/apimodels/reports.py | 70 ++ apiserver/bll/event/event_bll.py | 11 +- apiserver/bll/event/event_metrics.py | 2 + apiserver/bll/task/task_bll.py | 7 +- apiserver/database/model/task/task.py | 1 + apiserver/schema/services/_common.conf | 29 + apiserver/schema/services/_events_common.conf | 106 +++ apiserver/schema/services/_tasks_common.conf | 498 +++++++++++++ apiserver/schema/services/events.conf | 161 +--- apiserver/schema/services/reports.conf | 686 ++++++++++++++++++ apiserver/schema/services/tasks.conf | 572 +-------------- apiserver/services/events.py | 141 ++-- apiserver/services/reports.py | 387 ++++++++++ apiserver/services/tasks.py | 10 +- apiserver/tests/automated/__init__.py | 4 +- apiserver/tests/automated/test_reports.py | 189 +++++ 17 files changed, 2122 insertions(+), 755 deletions(-) create mode 100644 apiserver/apimodels/reports.py create mode 100644 apiserver/schema/services/_events_common.conf create mode 100644 apiserver/schema/services/_tasks_common.conf create mode 100644 apiserver/schema/services/reports.conf create mode 100644 apiserver/services/reports.py create mode 100644 apiserver/tests/automated/test_reports.py diff --git a/apiserver/apierrors/errors.conf b/apiserver/apierrors/errors.conf index b456f49..01db053 100644 --- a/apiserver/apierrors/errors.conf +++ b/apiserver/apierrors/errors.conf @@ -50,6 +50,9 @@ 130: ["task_not_found", "task not found"] 131: ["events_not_added", "events not added"] + # Reports + 150: ["operation_supported_on_reports_only", "passed task is not report"] + # Models 200: ["model_error", "general task error"] 201: ["invalid_model_id", "invalid model id"] diff --git a/apiserver/apimodels/reports.py b/apiserver/apimodels/reports.py new file mode 100644 index 0000000..6537807 --- /dev/null +++ b/apiserver/apimodels/reports.py @@ -0,0 +1,70 @@ +from typing import Sequence + +from jsonmodels import validators +from jsonmodels.fields import StringField, ListField, BoolField, EmbeddedField, IntField +from jsonmodels.models import Base +from jsonmodels.validators import Length + +from apiserver.apimodels.events import MetricVariants, HistogramRequestBase + + +class UpdateReportRequest(Base): + task = StringField(required=True) + name = StringField(nullable=True, validators=Length(minimum_value=3)) + tags = ListField(items_types=[str]) + comment = StringField() + report = StringField() + + +class CreateReportRequest(Base): + name = StringField(required=True, validators=Length(minimum_value=3)) + tags = ListField(items_types=[str]) + comment = StringField() + report = StringField() + project = StringField() + + +class PublishReportRequest(Base): + task = StringField(required=True) + message = StringField(default="") + + +class ArchiveReportRequest(Base): + task = StringField(required=True) + message = StringField(default="") + + +class ShareReportRequest(Base): + task = StringField(required=True) + share = BoolField(default=True) + + +class DeleteReportRequest(Base): + task = StringField(required=True) + force = BoolField(default=False) + + +class MoveReportRequest(Base): + task = StringField(required=True) + project = StringField() + project_name = StringField() + + +class EventsRequest(Base): + iters = IntField(default=1, validators=validators.Min(1)) + metrics: Sequence[MetricVariants] = ListField(items_types=MetricVariants) + + +class ScalarMetricsIterHistogram(HistogramRequestBase): + metrics: Sequence[MetricVariants] = ListField(items_types=MetricVariants) + + +class GetTasksDataRequest(Base): + debug_images: EventsRequest = EmbeddedField(EventsRequest) + plots: EventsRequest = EmbeddedField(EventsRequest) + scalar_metrics_iter_histogram: ScalarMetricsIterHistogram = EmbeddedField(ScalarMetricsIterHistogram) + allow_public = BoolField(default=True) + + +class GetAllRequest(Base): + allow_public = BoolField(default=True) diff --git a/apiserver/bll/event/event_bll.py b/apiserver/bll/event/event_bll.py index 0108064..bd47e43 100644 --- a/apiserver/bll/event/event_bll.py +++ b/apiserver/bll/event/event_bll.py @@ -766,10 +766,9 @@ class EventBLL(object): def get_task_events( self, company_id: str, - task_id: str, + task_id: Union[str, Sequence[str]], event_type: EventType, - metric=None, - variant=None, + metrics: MetricVariants = None, last_iter_count=None, sort=None, size=500, @@ -790,10 +789,8 @@ class EventBLL(object): task_ids = [task_id] if isinstance(task_id, str) else task_id must = [] - if metric: - must.append({"term": {"metric": metric}}) - if variant: - must.append({"term": {"variant": variant}}) + if metrics: + must.append(get_metric_variants_condition(metrics)) if last_iter_count is None or model_events: must.append({"terms": {"task": task_ids}}) diff --git a/apiserver/bll/event/event_metrics.py b/apiserver/bll/event/event_metrics.py index 7b5dba3..4c3c616 100644 --- a/apiserver/bll/event/event_metrics.py +++ b/apiserver/bll/event/event_metrics.py @@ -112,6 +112,7 @@ class EventMetrics: tasks: Sequence[Task], samples, key: ScalarKeyEnum, + metric_variants: MetricVariants = None, ): """ Compare scalar metrics for different tasks per metric and variant @@ -128,6 +129,7 @@ class EventMetrics: event_type=event_type, samples=samples, key=ScalarKey.resolve(key), + metric_variants=metric_variants, run_parallel=False, ) task_ids = [t.id for t in tasks] diff --git a/apiserver/bll/task/task_bll.py b/apiserver/bll/task/task_bll.py index ae181ab..ce2b806 100644 --- a/apiserver/bll/task/task_bll.py +++ b/apiserver/bll/task/task_bll.py @@ -128,13 +128,12 @@ class TaskBLL: return list(q) @staticmethod - def create(call: APICall, fields: dict): - identity = call.identity + def create(company: str, user: str, fields: dict): now = datetime.utcnow() return Task( id=create_id(), - user=identity.user, - company=identity.company, + user=user, + company=company, created=now, last_update=now, last_change=now, diff --git a/apiserver/database/model/task/task.py b/apiserver/database/model/task/task.py index e6277ff..99911f5 100644 --- a/apiserver/database/model/task/task.py +++ b/apiserver/database/model/task/task.py @@ -149,6 +149,7 @@ class TaskType(object): application = "application" monitor = "monitor" controller = "controller" + report = "report" optimizer = "optimizer" service = "service" qc = "qc" diff --git a/apiserver/schema/services/_common.conf b/apiserver/schema/services/_common.conf index 64723f9..590a578 100644 --- a/apiserver/schema/services/_common.conf +++ b/apiserver/schema/services/_common.conf @@ -15,6 +15,35 @@ metadata_item { } } } +task_status_enum { + type: string + enum: [ + created + queued + in_progress + stopped + published + publishing + closed + failed + completed + unknown + ] +} +multi_field_pattern_data { + type: object + properties { + pattern { + description: "Pattern string (regex)" + type: string + } + fields { + description: "List of field names" + type: array + items { type: string } + } + } +} credentials { type: object properties { diff --git a/apiserver/schema/services/_events_common.conf b/apiserver/schema/services/_events_common.conf new file mode 100644 index 0000000..bec8aa6 --- /dev/null +++ b/apiserver/schema/services/_events_common.conf @@ -0,0 +1,106 @@ +scalar_key_enum { + type: string + enum: [ + iter + timestamp + iso_time + ] +} +metric_variants { + type: object + properties { + metric { + description: The metric name + type: string + } + variants { + type: array + description: The names of the metric variants + items {type: string} + } + } +} +debug_images_response_task_metrics { + type: object + properties { + task { + type: string + description: Task ID + } + iterations { + type: array + items { + type: object + properties { + iter { + type: integer + description: Iteration number + } + events { + type: array + items { + type: object + description: Debug image event + } + } + } + } + } + } +} +debug_images_response { + type: object + properties { + scroll_id { + type: string + description: "Scroll ID for getting more results" + } + metrics { + type: array + description: "Debug image events grouped by tasks and iterations" + items {"$ref": "#/definitions/debug_images_response_task_metrics"} + } + } +} +plots_response_task_metrics { + type: object + properties { + task { + type: string + description: Task ID + } + iterations { + type: array + items { + type: object + properties { + iter { + type: integer + description: Iteration number + } + events { + type: array + items { + type: object + description: Plot event + } + } + } + } + } + } +} +plots_response { + type: object + properties { + scroll_id { + type: string + description: "Scroll ID for getting more results" + } + metrics { + type: array + description: "Plot events grouped by tasks and iterations" + items {"$ref": "#/definitions/plots_response_task_metrics"} + } + } +} \ No newline at end of file diff --git a/apiserver/schema/services/_tasks_common.conf b/apiserver/schema/services/_tasks_common.conf new file mode 100644 index 0000000..b24a10b --- /dev/null +++ b/apiserver/schema/services/_tasks_common.conf @@ -0,0 +1,498 @@ +include "_common.conf" +task_type_enum { + type: string + enum: [ + training + testing + inference + data_processing + application + monitor + controller + optimizer + service + qc + custom + ] +} +script { + type: object + properties { + binary { + description: "Binary to use when running the script" + type: string + default: python + } + repository { + description: "Name of the repository where the script is located" + type: string + } + tag { + description: "Repository tag" + type: string + } + branch { + description: "Repository branch id If not provided and tag not provided, default repository branch is used." + type: string + } + version_num { + description: "Version (changeset) number. Optional (default is head version) Unused if tag is provided." + type: string + } + entry_point { + description: "Path to execute within the repository" + type: string + } + working_dir { + description: "Path to the folder from which to run the script Default - root folder of repository" + type: string + } + requirements { + description: "A JSON object containing requirements strings by key" + type: object + } + diff { + description: "Uncommitted changes found in the repository when task was run" + type: string + } + } +} +model_type_enum { + type: string + enum: ["input", "output"] +} +task_model_item { + type: object + required: [ name, model] + properties { + name { + description: "The task model name" + type: string + } + model { + description: "The model ID" + type: string + } + } +} +output { + type: object + properties { + destination { + description: "Storage id. This is where output files will be stored." + type: string + } + model { + description: "Model id." + type: string + } + result { + description: "Task result. Values: 'success', 'failure'" + type: string + } + error { + description: "Last error text" + type: string + } + } + } +task_execution_progress_enum { + type: string + enum: [ + unknown + running + stopping + stopped + ] +} +artifact_type_data { + type: object + properties { + preview { + description: "Description or textual data" + type: string + } + content_type { + description: "System defined raw data content type" + type: string + } + data_hash { + description: "Hash of raw data, without any headers or descriptive parts" + type: string + } + } +} +artifact_mode_enum { + type: string + enum: [ + input + output + ] + default: output +} +artifact { + type: object + required: [key, type] + properties { + key { + description: "Entry key" + type: string + } + type { + description: "System defined type" + type: string + } + mode { + description: "System defined input/output indication" + "$ref": "#/definitions/artifact_mode_enum" + } + uri { + description: "Raw data location" + type: string + } + content_size { + description: "Raw data length in bytes" + type: integer + } + hash { + description: "Hash of entire raw data" + type: string + } + timestamp { + description: "Epoch time when artifact was created" + type: integer + } + type_data { + description: "Additional fields defined by the system" + "$ref": "#/definitions/artifact_type_data" + } + display_data { + description: "User-defined list of key/value pairs, sorted" + type: array + items { + type: array + items { + type: string # can also be a number... TODO: upgrade the generator + } + } + } + } +} +artifact_id { + type: object + required: [key] + properties { + key { + description: "Entry key" + type: string + } + mode { + description: "System defined input/output indication" + "$ref": "#/definitions/artifact_mode_enum" + } + } +} +task_models { + type: object + properties { + input { + description: "The list of task input models" + type: array + items {"$ref": "#/definitions/task_model_item"} + + } + output { + description: "The list of task output models" + type: array + items {"$ref": "#/definitions/task_model_item"} + } + } +} +execution { + type: object + properties { + queue { + description: "Queue ID where task was queued." + type: string + } + parameters { + description: "Json object containing the Task parameters" + type: object + additionalProperties: true + } + model { + description: "Execution input model ID Not applicable for Register (Import) tasks" + type: string + } + model_desc { + description: "Json object representing the Model descriptors" + type: object + additionalProperties: true + } + model_labels { + description: """Json object representing the ids of the labels in the model. + The keys are the layers' names and the values are the IDs. + Not applicable for Register (Import) tasks. + Mandatory for Training tasks""" + type: object + additionalProperties: { type: integer } + } + framework { + description: """Framework related to the task. Case insensitive. Mandatory for Training tasks. """ + type: string + } + docker_cmd { + description: "Command for running docker script for the execution of the task" + type: string + } + artifacts { + description: "Task artifacts" + type: array + items { "$ref": "#/definitions/artifact" } + } + } +} +last_metrics_event { + type: object + properties { + metric { + description: "Metric name" + type: string + } + variant { + description: "Variant name" + type: string + } + value { + description: "Last value reported" + type: number + } + min_value { + description: "Minimum value reported" + type: number + } + max_value { + description: "Maximum value reported" + type: number + } + } +} +last_metrics_variants { + type: object + description: "Last metric events, one for each variant hash" + additionalProperties { + "$ref": "#/definitions/last_metrics_event" + } +} +params_item { + type: object + properties { + section { + description: "Section that the parameter belongs to" + type: string + } + name { + description: "Name of the parameter. The combination of section and name should be unique" + type: string + } + value { + description: "Value of the parameter" + type: string + } + type { + description: "Type of the parameter. Optional" + type: string + } + description { + description: "The parameter description. Optional" + type: string + } + } +} +configuration_item { + type: object + properties { + name { + description: "Name of the parameter. Should be unique" + type: string + } + value { + description: "Value of the parameter" + type: string + } + type { + description: "Type of the parameter. Optional" + type: string + } + description { + description: "The parameter description. Optional" + type: string + } + } +} +section_params { + description: "Task section params" + type: object + additionalProperties { + "$ref": "#/definitions/params_item" + } +} +task { + type: object + properties { + id { + description: "Task id" + type: string + } + name { + description: "Task Name" + type: string + } + user { + description: "Associated user id" + type: string + } + company { + description: "Company ID" + type: string + } + type { + description: "Type of task. Values: 'training', 'testing'" + "$ref": "#/definitions/task_type_enum" + } + status { + description: "" + "$ref": "#/definitions/task_status_enum" + } + comment { + description: "Free text comment" + type: string + } + created { + description: "Task creation time (UTC) " + type: string + format: "date-time" + } + started { + description: "Task start time (UTC)" + type: string + format: "date-time" + } + completed { + description: "Task end time (UTC)" + type: string + format: "date-time" + } + active_duration { + description: "Task duration time (seconds)" + type: integer + } + parent { + description: "Parent task id" + type: string + } + project { + description: "Project ID of the project to which this task is assigned" + type: string + } + output { + description: "Task output params" + "$ref": "#/definitions/output" + } + execution { + description: "Task execution params" + "$ref": "#/definitions/execution" + } + container { + description: "Docker container parameters" + type: object + additionalProperties { type: [string, null] } + } + models { + description: "Task models" + "$ref": "#/definitions/task_models" + } + // TODO: will be removed + script { + description: "Script info" + "$ref": "#/definitions/script" + } + tags { + description: "User-defined tags list" + type: array + items { type: string } + } + system_tags { + description: "System tags list. This field is reserved for system use, please don't use it." + type: array + items { type: string } + } + status_changed { + description: "Last status change time" + type: string + format: "date-time" + } + status_message { + description: "free text string representing info about the status" + type: string + } + status_reason { + description: "Reason for last status change" + type: string + } + published { + description: "Task publish time" + type: string + format: "date-time" + } + last_worker { + description: "ID of last worker that handled the task" + type: string + } + last_worker_report { + description: "Last time a worker reported while working on this task" + type: string + format: "date-time" + } + last_update { + description: "Last time this task was created, edited, changed or events for this task were reported" + type: string + format: "date-time" + } + last_change { + description: "Last time any update was done to the task" + type: string + format: "date-time" + } + last_iteration { + description: "Last iteration reported for this task" + type: integer + } + last_metrics { + description: "Last metric variants (hash to events), one for each metric hash" + type: object + additionalProperties { + "$ref": "#/definitions/last_metrics_variants" + } + } + hyperparams { + description: "Task hyper params per section" + type: object + additionalProperties { + "$ref": "#/definitions/section_params" + } + } + configuration { + description: "Task configuration params" + type: object + additionalProperties { + "$ref": "#/definitions/configuration_item" + } + } + runtime { + description: "Task runtime mapping" + type: object + additionalProperties: true + } + } +} diff --git a/apiserver/schema/services/events.conf b/apiserver/schema/services/events.conf index 0e58f8d..7944d81 100644 --- a/apiserver/schema/services/events.conf +++ b/apiserver/schema/services/events.conf @@ -1,17 +1,6 @@ _description : "Provides an API for running tasks to report events collected by the system." _definitions { - metric_variants { - type: object - metric { - description: The metric name - type: string - } - variants { - type: array - description: The names of the metric variants - items {type: string} - } - } + include "_events_common.conf" metrics_scalar_event { description: "Used for reporting scalar metrics during training task" type: object @@ -164,14 +153,6 @@ _definitions { } } } - scalar_key_enum { - type: string - enum: [ - iter - timestamp - iso_time - ] - } log_level_enum { type: string enum: [ @@ -260,90 +241,6 @@ _definitions { } } } - debug_images_response_task_metrics { - type: object - properties { - task { - type: string - description: Task ID - } - iterations { - type: array - items { - type: object - properties { - iter { - type: integer - description: Iteration number - } - events { - type: array - items { - type: object - description: Debug image event - } - } - } - } - } - } - } - debug_images_response { - type: object - properties { - scroll_id { - type: string - description: "Scroll ID for getting more results" - } - metrics { - type: array - description: "Debug image events grouped by tasks and iterations" - items {"$ref": "#/definitions/debug_images_response_task_metrics"} - } - } - } - plots_response_task_metrics { - type: object - properties { - task { - type: string - description: Task ID - } - iterations { - type: array - items { - type: object - properties { - iter { - type: integer - description: Iteration number - } - events { - type: array - items { - type: object - description: Plot event - } - } - } - } - } - } - } - plots_response { - type: object - properties { - scroll_id { - type: string - description: "Scroll ID for getting more results" - } - metrics { - type: array - description: "Plot events grouped by tasks and iterations" - items {"$ref": "#/definitions/plots_response_task_metrics"} - } - } - } debug_image_sample_response { type: object properties { @@ -547,7 +444,7 @@ debug_images { } total { type: number - description: "Total number of results available for this query" + description: "Total number of results available for this query. In case there are more than 10000 results it is set to 10000" } scroll_id { type: string @@ -601,7 +498,7 @@ debug_images { } } "2.22": ${debug_images."2.14"} { - model_events { + request.properties.model_events { type: boolean description: If set then the retrieving model events. Otherwise task events default: false @@ -624,7 +521,7 @@ plots { } iters { type: integer - description: "Max number of latest iterations for which to return debug images" + description: "Max number of latest iterations for which to return plots" } navigate_earlier { type: boolean @@ -643,7 +540,7 @@ plots { response {"$ref": "#/definitions/plots_response"} } "2.22": ${plots."2.20"} { - model_events { + request.properties.model_events { type: boolean description: If set then the retrieving model plots. Otherwise task plots default: false @@ -693,7 +590,7 @@ get_debug_image_sample { } } "2.22": ${get_debug_image_sample."2.20"} { - model_events { + request.properties.model_events { type: boolean description: If set then the retrieving model debug images. Otherwise task debug images default: false @@ -730,7 +627,7 @@ next_debug_image_sample { default: false description: If set then navigate to the next/previous iteration } - model_events { + request.properties.model_events { type: boolean description: If set then the retrieving model debug images. Otherwise task debug images default: false @@ -774,7 +671,7 @@ get_plot_sample { response {"$ref": "#/definitions/plot_sample_response"} } "2.22": ${get_plot_sample."2.20"} { - model_events { + request.properties.model_events { type: boolean description: If set then the retrieving model plots. Otherwise task plots default: false @@ -811,7 +708,7 @@ next_plot_sample { default: false description: If set then navigate to the next/previous iteration } - model_events { + request.properties.model_events { type: boolean description: If set then the retrieving model plots. Otherwise task plots default: false @@ -850,7 +747,7 @@ get_task_metrics{ } } "2.22": ${get_task_metrics."2.7"} { - model_events { + request.properties.model_events { type: boolean description: If set then get metrics from model events. Otherwise from task events default: false @@ -955,7 +852,7 @@ get_task_log { } total { type: number - description: "Total number of results available for this query" + description: "Total number of results available for this query. In case there are more than 10000 results it is set to 10000" } scroll_id { type: string @@ -1009,7 +906,7 @@ get_task_log { } total { type: number - description: "Total number of log events available for this query" + description: "Total number of log events available for this query. In case there are more than 10000 events it is set to 10000" } } } @@ -1064,7 +961,7 @@ get_task_events { } total { type: number - description: "Total number of results available for this query" + description: "Total number of results available for this query. In case there are more than 10000 results it is set to 10000" } scroll_id { type: string @@ -1074,7 +971,7 @@ get_task_events { } } "2.22": ${get_task_events."2.1"} { - model_events { + request.properties.model_events { type: boolean description: If set then get retrieving model events. Otherwise task events default: false @@ -1154,7 +1051,7 @@ get_task_plots { } total { type: number - description: "Total number of results available for this query" + description: "Total number of results available for this query. In case there are more than 10000 results it is set to 10000" } scroll_id { type: string @@ -1182,7 +1079,7 @@ get_task_plots { } } "2.22": ${get_task_plots."2.16"} { - model_events { + request.properties.model_events { type: boolean description: If set then the retrieving model events. Otherwise task events default: false @@ -1208,7 +1105,7 @@ get_multi_task_plots { } iters { type: integer - description: "Max number of latest iterations for which to return debug images" + description: "Max number of latest iterations for which to return plots" } scroll_id { type: string @@ -1229,7 +1126,7 @@ get_multi_task_plots { } total { type: number - description: "Total number of results available for this query" + description: "Total number of results available for this query. In case there are more than 10000 results it is set to 10000" } scroll_id { type: string @@ -1246,7 +1143,7 @@ get_multi_task_plots { } } "2.22": ${get_multi_task_plots."2.16"} { - model_events { + request.properties.model_events { type: boolean description: If set then the retrieving model events. Otherwise task events default: false @@ -1281,7 +1178,7 @@ get_vector_metrics_and_variants { } } "2.22": ${get_vector_metrics_and_variants."2.1"} { - model_events { + request.properties.model_events { type: boolean description: If set then the retrieving model events. Otherwise task events default: false @@ -1326,7 +1223,7 @@ vector_metrics_iter_histogram { } } "2.22": ${vector_metrics_iter_histogram."2.1"} { - model_events { + request.properties.model_events { type: boolean description: If set then the retrieving model events. Otherwise task events default: false @@ -1386,7 +1283,7 @@ scalar_metrics_iter_histogram { } } "2.22": ${scalar_metrics_iter_histogram."2.14"} { - model_events { + request.properties.model_events { type: boolean description: If set then the retrieving model events. Otherwise task events default: false @@ -1403,7 +1300,7 @@ multi_task_scalar_metrics_iter_histogram { ] properties { tasks { - description: "List of task Task IDs. Maximum amount of tasks is 10" + description: "List of task Task IDs. Maximum amount of tasks is 100" type: array items { type: string @@ -1432,7 +1329,7 @@ multi_task_scalar_metrics_iter_histogram { } } "2.22": ${multi_task_scalar_metrics_iter_histogram."2.1"} { - model_events { + request.properties.model_events { type: boolean description: If set then the retrieving model events. Otherwise task events default: false @@ -1488,7 +1385,7 @@ get_task_single_value_metrics { } } "2.22": ${get_task_single_value_metrics."2.20"} { - model_events { + request.properties.model_events { type: boolean description: If set then the retrieving model events. Otherwise task events default: false @@ -1574,7 +1471,7 @@ get_scalar_metrics_and_variants { } } "2.22": ${get_scalar_metrics_and_variants."2.1"} { - model_events { + request.properties.model_events { type: boolean description: If set then the retrieving model events. Otherwise task events default: false @@ -1630,7 +1527,7 @@ get_scalar_metric_data { } } "2.22": ${get_scalar_metric_data."2.16"} { - model_events { + request.properties.model_events { type: boolean description: If set then the retrieving model events. Otherwise task events default: false @@ -1701,7 +1598,7 @@ scalar_metrics_iter_raw { } } "2.22": ${scalar_metrics_iter_raw."2.16"} { - model_events { + request.properties.model_events { type: boolean description: If set then the retrieving model events. Otherwise task events default: false @@ -1762,4 +1659,4 @@ clear_task_log { } } } -} \ No newline at end of file +} diff --git a/apiserver/schema/services/reports.conf b/apiserver/schema/services/reports.conf new file mode 100644 index 0000000..9610451 --- /dev/null +++ b/apiserver/schema/services/reports.conf @@ -0,0 +1,686 @@ +_description: "Provides a management API for reports in the system." +_definitions { + include "_tasks_common.conf" + include "_events_common.conf" + update_response { + type: object + properties { + updated { + description: "Number of reports updated (0 or 1)" + type: integer + enum: [ 0, 1 ] + } + fields { + description: "Updated fields names and values" + type: object + additionalProperties: true + } + } + } + report_status_enum { + type: string + enum: [ + created + published + ] + } + report { + type: object + properties { + id { + description: "Report id" + type: string + } + name { + description: "Report Name" + type: string + } + user { + description: "Associated user id" + type: string + } + company { + description: "Company ID" + type: string + } + status { + description: "" + "$ref": "#/definitions/report_status_enum" + } + comment { + description: "Free text comment" + type: string + } + report { + description: "Report template" + type: string + } + created { + description: "Report creation time (UTC) " + type: string + format: "date-time" + } + project { + description: "Project ID of the project to which this report is assigned" + type: string + } + tags { + description: "User-defined tags list" + type: array + items { type: string } + } + system_tags { + description: "System tags list. This field is reserved for system use, please don't use it." + type: array + items { type: string } + } + status_changed { + description: "Last status change time" + type: string + format: "date-time" + } + status_message { + description: "free text string representing info about the status" + type: string + } + status_reason { + description: "Reason for last status change" + type: string + } + published { + description: "Report publish time" + type: string + format: "date-time" + } + last_update { + description: "Last time this report was created, edited, changed" + type: string + format: "date-time" + } + } + } +} +create { + "999.0" { + description: "Create a new report" + request { + type: object + required: [ + name + ] + properties { + name { + description: "Report name. Unique within the company." + type: string + } + tags { + description: "User-defined tags list" + type: array + items { type: string } + } + comment { + description: "Free text comment " + type: string + } + report { + description: "Report template" + type: string + } + project { + description: "Project ID of the project to which this report is assigned Must exist[ab]" + type: string + } + } + } + response { + type: object + properties { + id { + description: "ID of the report" + type: string + } + } + } + } +} +update { + "999.0" { + description: "Create a new report" + request { + type: object + required: [ + task + ] + properties { + task { + description: "The ID of the report task to update" + type: string + } + name { + description: "Report name. Unique within the company." + type: string + } + tags { + description: "User-defined tags list" + type: array + items { type: string } + } + comment { + description: "Free text comment " + type: string + } + report { + description: "Report template" + type: string + } + } + } + response: ${_definitions.update_response} + } +} +move { + "999.0" { + description: "Move reports to a project" + request { + type: object + required: [task] + properties { + task { + description: "ID of the report to move" + type: string + } + project { + description: "Target project ID. If not provided, `project_name` must be provided." + type: string + } + project_name { + description: "Target project name. If provided and a project with this name does not exist, a new project will be created. If not provided, `project` must be provided." + type: string + } + } + } + response { + type: object + properties { + project_id: { + description: The ID of the target project + type: string + } + } + } + } +} +publish { + "999.0" { + description: "Publish report" + request { + type: object + required: [ + task + ] + properties { + task { + description: "The ID of the report task to publish" + type: string + } + comment { + description: "The client message" + type: string + } + } + } + response: ${_definitions.update_response} + } +} +archive { + "999.0" { + description: "Archive report" + request { + type: object + required: [ + task + ] + properties { + task { + description: "The ID of the report task to archive" + type: string + } + comment { + description: "The client message" + type: string + } + } + } + response { + type: object + properties { + archived { + description: "Number of reports archived (0 or 1)" + type: integer + enum: [0, 1] + } + } + } + } +} +unarchive { + "999.0" { + description: "Unarchive report" + request { + type: object + required: [ + task + ] + properties { + task { + description: "The ID of the report task to unarchive" + type: string + } + comment { + description: "The client message" + type: string + } + } + } + response { + type: object + properties { + unarchived { + description: "Number of reports unarchived (0 or 1)" + type: integer + enum: [0, 1] + } + } + } + } +} +//share { +// "999.0" { +// description: "Share or unshare report" +// request { +// type: object +// required: [ +// task +// ] +// properties { +// task { +// description: "The ID of the report task to share/unshare" +// type: string +// } +// share { +// description: "If set to 'true' then the report will be shared. Otherwise unshared." +// type: boolean +// default: true +// } +// } +// } +// response { +// type: object +// properties { +// changed { +// description: "Number of changed reports (0 or 1)" +// type: integer +// enum: [0, 1] +// } +// } +// } +// } +//} +delete { + "999.0" { + description: "Delete report" + request { + type: object + required: [ + task + ] + properties { + task { + description: "The ID of the report task to delete" + type: string + } + force { + description: "If not set then published or unarchived reports cannot be deleted" + type: boolean + default: false + } + } + } + response { + type: object + properties { + deleted { + description: "Number of deleted reports (0 or 1)" + type: integer + enum: [0, 1] + } + } + } + } +} +get_task_data { + "999.0" { + description: "Get the tasks data according the passed search criteria + requested events" + request { + type: object + properties { + id { + description: "List of IDs to filter by" + type: array + items { type: string } + } + name { + description: "Get only tasks whose name matches this pattern (python regular expression syntax)" + type: string + } + user { + description: "List of user IDs used to filter results by the task's creating user" + type: array + items { type: string } + } + size { + type: integer + minimum: 1 + description: "The number of tasks to retrieve" + } + order_by { + description: "List of field names to order by. When search_text is used, '@text_score' can be used as a field representing the text score of returned documents. Use '-' prefix to specify descending order. Optional, recommended when using page" + type: array + items { type: string } + } + type { + description: "List of task types. One or more of: 'import', 'annotation', 'training' or 'testing' (case insensitive)" + type: array + items { type: string } + } + tags { + description: "List of task user-defined tags. Use '-' prefix to exclude tags" + type: array + items { type: string } + } + system_tags { + description: "List of task system tags. Use '-' prefix to exclude system tags" + type: array + items { type: string } + } + status { + description: "List of task status." + type: array + items { "$ref": "#/definitions/task_status_enum" } + } + project { + description: "List of project IDs" + type: array + items { type: string } + } + only_fields { + description: "List of task field names (nesting is supported using '.', e.g. execution.model_labels). If provided, this list defines the query's projection (only these fields will be returned for each result entry)" + type: array + items { type: string } + } + parent { + description: "Parent ID" + type: string + } + status_changed { + description: "List of status changed constraint strings (utcformat, epoch) with an optional prefix modifier (>, >=, <, <=)" + type: array + items { + type: string + pattern: "^(>=|>|<=|<)?.*$" + } + } + search_text { + description: "Free text search query" + type: string + } + allow_public { + description: "Allow public tasks to be returned in the results" + type: boolean + default: true + } + _all_ { + description: "Multi-field pattern condition (all fields match pattern)" + "$ref": "#/definitions/multi_field_pattern_data" + } + _any_ { + description: "Multi-field pattern condition (any field matches pattern)" + "$ref": "#/definitions/multi_field_pattern_data" + } + "input.view.entries.dataset" { + description: "List of input dataset IDs" + type: array + items { type: string } + } + "input.view.entries.version" { + description: "List of input dataset version IDs" + type: array + items { type: string } + } + search_hidden { + description: "If set to 'true' then hidden tasks are included in the search results" + type: boolean + default: false + } + include_subprojects { + description: "If set to 'true' and project field is set then tasks from the subprojects are searched too" + type: boolean + default: false + } + plots { + type: object + properties { + iters { + type: integer + description: "Max number of latest iterations for which to return plots" + } + metrics { + type: array + description: List of metrics and variants + items { "$ref": "#/definitions/metric_variants" } + } + } + } + debug_images { + type: object + properties { + iters { + type: integer + description: "Max number of latest iterations for which to return debug images" + } + metrics { + type: array + description: List of metrics and variants + items { "$ref": "#/definitions/metric_variants" } + } + } + } + scalar_metrics_iter_histogram { + type: object + properties { + samples { + description: "The amount of histogram points to return (0 to return all the points). Optional, the default value is 6000." + type: integer + } + key { + description: """ + Histogram x axis to use: + iter - iteration number + iso_time - event time as ISO formatted string + timestamp - event timestamp as milliseconds since epoch + """ + "$ref": "#/definitions/scalar_key_enum" + } + metrics { + type: array + description: List of metrics and variants + items { "$ref": "#/definitions/metric_variants" } + } + } + } + } + } + response { + type: object + properties { + tasks { + description: "List of tasks" + type: array + items { "$ref": "#/definitions/task" } + } + plots { + type: object + description: "Plots mapped by metric, variant, task and iteration" + additionalProperties: true + } + debug_images { + type: array + description: "Debug image events grouped by tasks and iterations" + items {"$ref": "#/definitions/debug_images_response_task_metrics"} + } + scalar_metrics_iter_histogram { + type: object + additionalProperties: true + } + } + } + } +} +get_all_ex { + "999.0" { + description: "Get all the company's and public report tasks" + request { + type: object + properties { + id { + description: "List of IDs to filter by" + type: array + items { type: string } + } + name { + description: "Get only reports whose name matches this pattern (python regular expression syntax)" + type: string + } + user { + description: "List of user IDs used to filter results by the reports's creating user" + type: array + items { type: string } + } + page { + description: "Page number, returns a specific page out of the resulting list of reports" + type: integer + minimum: 0 + } + page_size { + description: "Page size, specifies the number of results returned in each page (last page may contain fewer results)" + type: integer + minimum: 1 + } + order_by { + description: "List of field names to order by. When search_text is used, '@text_score' can be used as a field representing the text score of returned documents. Use '-' prefix to specify descending order. Optional, recommended when using page" + type: array + items { type: string } + } + tags { + description: "List of report user-defined tags. Use '-' prefix to exclude tags" + type: array + items { type: string } + } + system_tags { + description: "List of report system tags. Use '-' prefix to exclude system tags" + type: array + items { type: string } + } + status { + description: "List of report status." + type: array + items { "$ref": "#/definitions/report_status_enum" } + } + project { + description: "List of project IDs" + type: array + items { type: string } + } + only_fields { + description: "List of report field names (nesting is supported using '.'). If provided, this list defines the query's projection (only these fields will be returned for each result entry)" + type: array + items { type: string } + } + status_changed { + description: "List of status changed constraint strings (utcformat, epoch) with an optional prefix modifier (>, >=, <, <=)" + type: array + items { + type: string + pattern: "^(>=|>|<=|<)?.*$" + } + } + search_text { + description: "Free text search query" + type: string + } + scroll_id { + type: string + description: "Scroll ID returned from the previos calls to get_all" + } + refresh_scroll { + type: boolean + description: "If set then all the data received with this scroll will be requeried" + } + size { + type: integer + minimum: 1 + description: "The number of tasks to retrieve" + } + allow_public { + description: "Allow public reports to be returned in the results" + type: boolean + default: true + } + _all_ { + description: "Multi-field pattern condition (all fields match pattern)" + "$ref": "#/definitions/multi_field_pattern_data" + } + _any_ { + description: "Multi-field pattern condition (any field matches pattern)" + "$ref": "#/definitions/multi_field_pattern_data" + } + } + dependencies { + page: [ page_size ] + } + } + response { + type: object + properties { + tasks { + description: "List of report tasks" + type: array + items { "$ref": "#/definitions/report" } + } + scroll_id { + type: string + description: "Scroll ID that can be used with the next calls to get_all to retrieve more data" + } + } + } + } +} +get_tags { + "999.0" { + description: "Get all the user tags used for the company reports" + request { + type: object + additionalProperties: false + } + response { + type: object + properties { + tags { + description: "The list of unique tag values" + type: array + items {type: string} + } + } + } + } +} diff --git a/apiserver/schema/services/tasks.conf b/apiserver/schema/services/tasks.conf index c5751eb..729548f 100644 --- a/apiserver/schema/services/tasks.conf +++ b/apiserver/schema/services/tasks.conf @@ -25,7 +25,7 @@ _references { } } _definitions { - include "_common.conf" + include "_tasks_common.conf" change_many_request: ${_definitions.batch_operation} { request { properties { @@ -69,374 +69,6 @@ _definitions { } } } - multi_field_pattern_data { - type: object - properties { - pattern { - description: "Pattern string (regex)" - type: string - } - fields { - description: "List of field names" - type: array - items { type: string } - } - } - } - model_type_enum { - type: string - enum: ["input", "output"] - } - task_model_item { - type: object - required: [ name, model] - properties { - name { - description: "The task model name" - type: string - } - model { - description: "The model ID" - type: string - } - } - } - script { - type: object - properties { - binary { - description: "Binary to use when running the script" - type: string - default: python - } - repository { - description: "Name of the repository where the script is located" - type: string - } - tag { - description: "Repository tag" - type: string - } - branch { - description: "Repository branch id If not provided and tag not provided, default repository branch is used." - type: string - } - version_num { - description: "Version (changeset) number. Optional (default is head version) Unused if tag is provided." - type: string - } - entry_point { - description: "Path to execute within the repository" - type: string - } - working_dir { - description: "Path to the folder from which to run the script Default - root folder of repository" - type: string - } - requirements { - description: "A JSON object containing requirements strings by key" - type: object - } - diff { - description: "Uncommitted changes found in the repository when task was run" - type: string - } - } - } - output { - type: object - properties { - destination { - description: "Storage id. This is where output files will be stored." - type: string - } - model { - description: "Model id." - type: string - } - result { - description: "Task result. Values: 'success', 'failure'" - type: string - } - error { - description: "Last error text" - type: string - } - } - } - task_execution_progress_enum { - type: string - enum: [ - unknown - running - stopping - stopped - ] - } - output_rois_enum { - type: string - enum: [ - all_in_frame - only_filtered - frame_per_roi - ] - } - artifact_type_data { - type: object - properties { - preview { - description: "Description or textual data" - type: string - } - content_type { - description: "System defined raw data content type" - type: string - } - data_hash { - description: "Hash of raw data, without any headers or descriptive parts" - type: string - } - } - } - artifact_mode_enum { - type: string - enum: [ - input - output - ] - default: output - } - artifact { - type: object - required: [key, type] - properties { - key { - description: "Entry key" - type: string - } - type { - description: "System defined type" - type: string - } - mode { - description: "System defined input/output indication" - "$ref": "#/definitions/artifact_mode_enum" - } - uri { - description: "Raw data location" - type: string - } - content_size { - description: "Raw data length in bytes" - type: integer - } - hash { - description: "Hash of entire raw data" - type: string - } - timestamp { - description: "Epoch time when artifact was created" - type: integer - } - type_data { - description: "Additional fields defined by the system" - "$ref": "#/definitions/artifact_type_data" - } - display_data { - description: "User-defined list of key/value pairs, sorted" - type: array - items { - type: array - items { - type: string # can also be a number... TODO: upgrade the generator - } - } - } - } - } - artifact_id { - type: object - required: [key] - properties { - key { - description: "Entry key" - type: string - } - mode { - description: "System defined input/output indication" - "$ref": "#/definitions/artifact_mode_enum" - } - } - } - task_models { - type: object - properties { - input { - description: "The list of task input models" - type: array - items {"$ref": "#/definitions/task_model_item"} - - } - output { - description: "The list of task output models" - type: array - items {"$ref": "#/definitions/task_model_item"} - } - } - } - execution { - type: object - properties { - queue { - description: "Queue ID where task was queued." - type: string - } - parameters { - description: "Json object containing the Task parameters" - type: object - additionalProperties: true - } - model { - description: "Execution input model ID Not applicable for Register (Import) tasks" - type: string - } - model_desc { - description: "Json object representing the Model descriptors" - type: object - additionalProperties: true - } - model_labels { - description: """Json object representing the ids of the labels in the model. - The keys are the layers' names and the values are the IDs. - Not applicable for Register (Import) tasks. - Mandatory for Training tasks""" - type: object - additionalProperties: { type: integer } - } - framework { - description: """Framework related to the task. Case insensitive. Mandatory for Training tasks. """ - type: string - } - docker_cmd { - description: "Command for running docker script for the execution of the task" - type: string - } - artifacts { - description: "Task artifacts" - type: array - items { "$ref": "#/definitions/artifact" } - } - } - } - task_status_enum { - type: string - enum: [ - created - queued - in_progress - stopped - published - publishing - closed - failed - completed - unknown - ] - } - task_type_enum { - type: string - enum: [ - training - testing - inference - data_processing - application - monitor - controller - optimizer - service - qc - custom - ] - } - last_metrics_event { - type: object - properties { - metric { - description: "Metric name" - type: string - } - variant { - description: "Variant name" - type: string - } - value { - description: "Last value reported" - type: number - } - min_value { - description: "Minimum value reported" - type: number - } - max_value { - description: "Maximum value reported" - type: number - } - } - } - last_metrics_variants { - type: object - description: "Last metric events, one for each variant hash" - additionalProperties { - "$ref": "#/definitions/last_metrics_event" - } - } - params_item { - type: object - properties { - section { - description: "Section that the parameter belongs to" - type: string - } - name { - description: "Name of the parameter. The combination of section and name should be unique" - type: string - } - value { - description: "Value of the parameter" - type: string - } - type { - description: "Type of the parameter. Optional" - type: string - } - description { - description: "The parameter description. Optional" - type: string - } - } - } - configuration_item { - type: object - properties { - name { - description: "Name of the parameter. Should be unique" - type: string - } - value { - description: "Value of the parameter" - type: string - } - type { - description: "Type of the parameter. Optional" - type: string - } - description { - description: "The parameter description. Optional" - type: string - } - } - } param_key { type: object properties { @@ -450,13 +82,6 @@ _definitions { } } } - section_params { - description: "Task section params" - type: object - additionalProperties { - "$ref": "#/definitions/params_item" - } - } replace_hyperparams_enum { type: string enum: [ @@ -465,165 +90,6 @@ _definitions { all ] } - task { - type: object - properties { - id { - description: "Task id" - type: string - } - name { - description: "Task Name" - type: string - } - user { - description: "Associated user id" - type: string - } - company { - description: "Company ID" - type: string - } - type { - description: "Type of task. Values: 'training', 'testing'" - "$ref": "#/definitions/task_type_enum" - } - status { - description: "" - "$ref": "#/definitions/task_status_enum" - } - comment { - description: "Free text comment" - type: string - } - created { - description: "Task creation time (UTC) " - type: string - format: "date-time" - } - started { - description: "Task start time (UTC)" - type: string - format: "date-time" - } - completed { - description: "Task end time (UTC)" - type: string - format: "date-time" - } - active_duration { - description: "Task duration time (seconds)" - type: integer - } - parent { - description: "Parent task id" - type: string - } - project { - description: "Project ID of the project to which this task is assigned" - type: string - } - output { - description: "Task output params" - "$ref": "#/definitions/output" - } - execution { - description: "Task execution params" - "$ref": "#/definitions/execution" - } - container { - description: "Docker container parameters" - type: object - additionalProperties { type: [string, null] } - } - models { - description: "Task models" - "$ref": "#/definitions/task_models" - } - // TODO: will be removed - script { - description: "Script info" - "$ref": "#/definitions/script" - } - tags { - description: "User-defined tags list" - type: array - items { type: string } - } - system_tags { - description: "System tags list. This field is reserved for system use, please don't use it." - type: array - items { type: string } - } - status_changed { - description: "Last status change time" - type: string - format: "date-time" - } - status_message { - description: "free text string representing info about the status" - type: string - } - status_reason { - description: "Reason for last status change" - type: string - } - published { - description: "Last status change time" - type: string - format: "date-time" - } - last_worker { - description: "ID of last worker that handled the task" - type: string - } - last_worker_report { - description: "Last time a worker reported while working on this task" - type: string - format: "date-time" - } - last_update { - description: "Last time this task was created, updated, changed or events for this task were reported" - type: string - format: "date-time" - } - last_change { - description: "Last time any update was done to the task" - type: string - format: "date-time" - } - last_iteration { - description: "Last iteration reported for this task" - type: integer - } - last_metrics { - description: "Last metric variants (hash to events), one for each metric hash" - type: object - additionalProperties { - "$ref": "#/definitions/last_metrics_variants" - } - } - hyperparams { - description: "Task hyper params per section" - type: object - additionalProperties { - "$ref": "#/definitions/section_params" - } - } - configuration { - description: "Task configuration params" - type: object - additionalProperties { - "$ref": "#/definitions/configuration_item" - } - } - runtime { - description: "Task runtime mapping" - type: object - additionalProperties: true - } - } - } task_urls { type: object properties { @@ -1227,6 +693,10 @@ validate { description: "Task execution params" "$ref": "#/definitions/execution" } + script { + description: "Script info" + "$ref": "#/definitions/script" + } hyperparams { description: "Task hyper params per section" type: object @@ -1241,10 +711,6 @@ validate { "$ref": "#/definitions/configuration_item" } } - script { - description: "Script info" - "$ref": "#/definitions/script" - } } } response { @@ -1392,6 +858,10 @@ edit { description: "Task execution params" "$ref": "#/definitions/execution" } + script { + description: "Script info" + "$ref": "#/definitions/script" + } hyperparams { description: "Task hyper params per section" type: object @@ -1406,10 +876,6 @@ edit { "$ref": "#/definitions/configuration_item" } } - script { - description: "Script info" - "$ref": "#/definitions/script" - } } } response: ${_definitions.update_response} @@ -1478,6 +944,10 @@ reset { description: "If set to 'true' then return the urls of the files that were uploaded by this task. Default value is 'false'" type: boolean } + delete_output_models { + description: "If set to 'true' then delete output models of this task that are not referenced by other tasks. Default value is 'true'" + type: boolean + } } } response { @@ -1665,6 +1135,10 @@ delete { description: "If set to 'true' then return the urls of the files that were uploaded by this task. Default value is 'false'" type: boolean } + delete_output_models { + description: "If set to 'true' then delete output models of this task that are not referenced by other tasks. Default value is 'true'" + type: boolean + } } } response { @@ -1736,12 +1210,12 @@ archive_many { type: string } } - response { - properties { - succeeded.items.properties.archived { - description: "Indicates whether the task was archived" - type: boolean - } + } + response { + properties { + succeeded.items.properties.archived { + description: "Indicates whether the task was archived" + type: boolean } } } diff --git a/apiserver/services/events.py b/apiserver/services/events.py index c59f542..19eb439 100644 --- a/apiserver/services/events.py +++ b/apiserver/services/events.py @@ -2,7 +2,7 @@ import itertools import math from collections import defaultdict from operator import itemgetter -from typing import Sequence, Optional, Union, Tuple +from typing import Sequence, Optional, Union, Tuple, Mapping import attr import jsonmodels.fields @@ -27,7 +27,9 @@ from apiserver.apimodels.events import ( ClearScrollRequest, ClearTaskLogRequest, SingleValueMetricsRequest, - GetVariantSampleRequest, GetMetricSamplesRequest, + GetVariantSampleRequest, + GetMetricSamplesRequest, + TaskMetric, ) from apiserver.bll.event import EventBLL from apiserver.bll.event.event_common import EventType, MetricVariants @@ -405,7 +407,7 @@ def get_scalar_metric_data(call, company_id, _): task_id, event_type=EventType.metrics_scalar, sort=[{"iter": {"order": "desc"}}], - metric=metric, + metrics={metric: []}, scroll_id=scroll_id, no_scroll=no_scroll, model_events=model_events, @@ -457,6 +459,7 @@ def scalar_metrics_iter_histogram( task_id=request.task, samples=request.samples, key=request.key, + metric_variants=_get_metric_variants_from_request(request.metrics), ) call.result.data = metrics @@ -546,10 +549,10 @@ def get_multi_task_plots_v1_7(call, company_id, _): scroll_id=scroll_id, ) - tasks = {t.id: t.name for t in tasks_or_models} + task_names = {t.id: t.name for t in tasks_or_models} return_events = _get_top_iter_unique_events_per_task( - result.events, max_iters=iters, tasks=tasks + result.events, max_iters=iters, task_names=task_names ) call.result.data = dict( @@ -560,6 +563,34 @@ def get_multi_task_plots_v1_7(call, company_id, _): ) +def _get_multitask_plots( + company: str, + tasks_or_models: Sequence[Task], + last_iters: int, + metrics: MetricVariants = None, + scroll_id=None, + no_scroll=True, + model_events=False, +) -> Tuple[dict, int, str]: + task_names = {t.id: t.name for t in tasks_or_models} + + result = event_bll.get_task_events( + company_id=company, + task_id=list(task_names), + event_type=EventType.metrics_plot, + metrics=metrics, + last_iter_count=last_iters, + sort=[{"iter": {"order": "desc"}}], + scroll_id=scroll_id, + no_scroll=no_scroll, + model_events=model_events, + ) + return_events = _get_top_iter_unique_events_per_task( + result.events, max_iters=last_iters, task_names=task_names + ) + return return_events, result.total_events, result.next_scroll_id + + @endpoint("events.get_multi_task_plots", min_version="1.8", required_fields=["tasks"]) def get_multi_task_plots(call, company_id, _): task_ids = call.data["tasks"] @@ -572,28 +603,19 @@ def get_multi_task_plots(call, company_id, _): company_id, task_ids, model_events ) - result = event_bll.get_task_events( - company, - task_ids, - event_type=EventType.metrics_plot, - sort=[{"iter": {"order": "desc"}}], - last_iter_count=iters, + return_events, total_events, next_scroll_id = _get_multitask_plots( + company=company, + tasks_or_models=tasks_or_models, + last_iters=iters, scroll_id=scroll_id, no_scroll=no_scroll, model_events=model_events, ) - - tasks = {t.id: t.name for t in tasks_or_models} - - return_events = _get_top_iter_unique_events_per_task( - result.events, max_iters=iters, tasks=tasks - ) - call.result.data = dict( plots=return_events, returned=len(return_events), - total=result.total_events, - scroll_id=result.next_scroll_id, + total=total_events, + scroll_id=next_scroll_id, ) @@ -674,22 +696,42 @@ def get_task_plots(call, company_id, request: TaskPlotsRequest): ) +def _task_metrics_dict_from_request(req_metrics: Sequence[TaskMetric]) -> dict: + task_metrics = defaultdict(dict) + for tm in req_metrics: + task_metrics[tm.task][tm.metric] = tm.variants + for metrics in task_metrics.values(): + if None in metrics: + metrics.clear() + + return task_metrics + + +def _get_metrics_response(metric_events: Sequence[tuple]) -> Sequence[MetricEvents]: + return [ + MetricEvents( + task=task, + iterations=[ + IterationEvents(iter=iteration["iter"], events=iteration["events"]) + for iteration in iterations + ], + ) + for (task, iterations) in metric_events + ] + + @endpoint( "events.plots", request_data_model=MetricEventsRequest, response_data_model=MetricEventsResponse, ) def task_plots(call, company_id, request: MetricEventsRequest): - task_metrics = defaultdict(dict) - for tm in request.metrics: - task_metrics[tm.task][tm.metric] = tm.variants - for metrics in task_metrics.values(): - if None in metrics: - metrics.clear() - + task_metrics = _task_metrics_dict_from_request(request.metrics) + task_ids = list(task_metrics) company, _ = _get_task_or_model_index_company( - company_id, task_ids=list(task_metrics), model_events=request.model_events + company_id, task_ids=task_ids, model_events=request.model_events ) + result = event_bll.plots_iterator.get_task_events( company_id=company, task_metrics=task_metrics, @@ -701,16 +743,7 @@ def task_plots(call, company_id, request: MetricEventsRequest): call.result.data_model = MetricEventsResponse( scroll_id=result.next_scroll_id, - metrics=[ - MetricEvents( - task=task, - iterations=[ - IterationEvents(iter=iteration["iter"], events=iteration["events"]) - for iteration in iterations - ], - ) - for (task, iterations) in result.metric_events - ], + metrics=_get_metrics_response(result.metric_events), ) @@ -789,15 +822,10 @@ def get_debug_images_v1_8(call, company_id, _): response_data_model=MetricEventsResponse, ) def get_debug_images(call, company_id, request: MetricEventsRequest): - task_metrics = defaultdict(dict) - for tm in request.metrics: - task_metrics[tm.task][tm.metric] = tm.variants - for metrics in task_metrics.values(): - if None in metrics: - metrics.clear() - + task_metrics = _task_metrics_dict_from_request(request.metrics) + task_ids = list(task_metrics) company, _ = _get_task_or_model_index_company( - company_id, task_ids=list(task_metrics), model_events=request.model_events + company_id, task_ids=task_ids, model_events=request.model_events ) result = event_bll.debug_images_iterator.get_task_events( @@ -811,16 +839,7 @@ def get_debug_images(call, company_id, request: MetricEventsRequest): call.result.data_model = MetricEventsResponse( scroll_id=result.next_scroll_id, - metrics=[ - MetricEvents( - task=task, - iterations=[ - IterationEvents(iter=iteration["iter"], events=iteration["events"]) - for iteration in iterations - ], - ) - for (task, iterations) in result.metric_events - ], + metrics=_get_metrics_response(result.metric_events), ) @@ -955,7 +974,9 @@ def clear_task_log(call: APICall, company_id: str, request: ClearTaskLogRequest) ) -def _get_top_iter_unique_events_per_task(events, max_iters, tasks): +def _get_top_iter_unique_events_per_task( + events, max_iters: int, task_names: Mapping[str, str] +): key = itemgetter("metric", "variant", "task", "iter") unique_events = itertools.chain.from_iterable( @@ -968,7 +989,7 @@ def _get_top_iter_unique_events_per_task(events, max_iters, tasks): def collect(evs, fields): if not fields: evs = list(evs) - return {"name": tasks.get(evs[0].get("task")), "plots": evs} + return {"name": task_names.get(evs[0].get("task")), "plots": evs} return { str(k): collect(group, fields[1:]) for k, group in itertools.groupby(evs, key=itemgetter(fields[0])) @@ -1034,7 +1055,9 @@ def scalar_metrics_iter_raw( request.batch_size = request.batch_size or scroll.request.batch_size task_id = request.task - task_or_model = _assert_task_or_model_exists(company_id, task_id, model_events=request.model_events)[0] + task_or_model = _assert_task_or_model_exists( + company_id, task_id, model_events=request.model_events + )[0] metric_variants = _get_metric_variants_from_request([request.metric]) if request.count_total and total is None: diff --git a/apiserver/services/reports.py b/apiserver/services/reports.py new file mode 100644 index 0000000..5cea332 --- /dev/null +++ b/apiserver/services/reports.py @@ -0,0 +1,387 @@ +import textwrap +from datetime import datetime +from typing import Sequence + +from apiserver.apimodels.reports import ( + CreateReportRequest, + UpdateReportRequest, + PublishReportRequest, + ArchiveReportRequest, + DeleteReportRequest, + MoveReportRequest, + GetTasksDataRequest, + EventsRequest, + GetAllRequest, +) +from apiserver.apierrors import errors +from apiserver.apimodels.base import IdResponse, UpdateResponse +from apiserver.services.utils import process_include_subprojects, sort_tags_response +from apiserver.bll.organization import OrgBLL +from apiserver.bll.project import ProjectBLL +from apiserver.bll.task import TaskBLL, ChangeStatusRequest +from apiserver.database.model import EntityVisibility +from apiserver.database.model.project import Project +from apiserver.database.model.task.task import Task, TaskType, TaskStatus +from apiserver.service_repo import APICall, endpoint +from apiserver.services.events import ( + _get_task_or_model_index_company, + event_bll, + _get_metrics_response, + _get_metric_variants_from_request, + _get_multitask_plots, +) +from apiserver.services.tasks import ( + escape_execution_parameters, + _hidden_query, + unprepare_from_saved, +) + +org_bll = OrgBLL() +project_bll = ProjectBLL() +task_bll = TaskBLL() + + +reports_project_name = ".reports" +reports_tag = "reports" +update_fields = { + "name", + "tags", + "comment", + "report", +} + + +def _assert_report( + company_id, task_id, only_fields=None, requires_write_access=True +): + if only_fields and "type" not in only_fields: + only_fields += ("type",) + + task = TaskBLL.get_task_with_access( + task_id=task_id, + company_id=company_id, + only=only_fields, + requires_write_access=requires_write_access, + ) + if task.type != TaskType.report: + raise errors.bad_request.OperationSupportedOnReportsOnly(id=task_id) + + return task + + +@endpoint("reports.update", response_data_model=UpdateResponse) +def update_report(call: APICall, company_id: str, request: UpdateReportRequest): + task = _assert_report( + task_id=request.task, + company_id=company_id, + only_fields=("status",), + ) + if task.status != TaskStatus.created: + raise errors.bad_request.InvalidTaskStatus( + expected=TaskStatus.created, status=task.status + ) + + partial_update_dict = { + field: value for field, value in call.data.items() if field in update_fields + } + if not partial_update_dict: + return UpdateResponse(updated=0) + + now = datetime.utcnow() + updated = task.update( + upsert=False, + **partial_update_dict, + last_change=now, + last_update=now, + last_changed_by=call.identity.user, + ) + if not updated: + return UpdateResponse(updated=0) + + updated_tags = partial_update_dict.get("tags") + if updated_tags: + partial_update_dict["tags"] = sorted(updated_tags) + updated_report = partial_update_dict.get("report") + if updated_report: + partial_update_dict["report"] = textwrap.shorten(updated_report, width=100) + + return UpdateResponse(updated=updated, fields=partial_update_dict) + + +def _ensure_reports_project(company: str, user: str, name: str): + name = name.strip("/") + _, _, basename = name.rpartition("/") + if basename != reports_project_name: + name = f"{name}/{reports_project_name}" + + return project_bll.find_or_create( + user=user, + company=company, + project_name=name, + description="Reports project", + system_tags=[reports_tag, EntityVisibility.hidden.value], + ) + + +@endpoint("reports.create", response_data_model=IdResponse) +def create_report(call: APICall, company_id: str, request: CreateReportRequest): + user_id = call.identity.user + project_id = request.project + if request.project: + project = Project.get_for_writing( + company=company_id, id=project_id, _only=("name",) + ) + project_name = project.name + else: + project_name = "" + + project_id = _ensure_reports_project( + company=company_id, user=user_id, name=project_name + ) + task = task_bll.create( + company=company_id, + user=user_id, + fields=dict( + project=project_id, + name=request.name, + tags=request.tags, + comment=request.comment, + type=TaskType.report, + system_tags=[reports_tag, EntityVisibility.hidden.value], + ), + ) + task.save() + return IdResponse(id=task.id) + + +def _delete_reports_project_if_empty(project_id): + project = Project.objects(id=project_id).only("basename").first() + if ( + project + and project.basename == reports_project_name + and Task.objects(project=project_id).count() == 0 + ): + project.delete() + + +@endpoint("reports.get_all_ex") +def get_all_ex(call: APICall, company_id, request: GetAllRequest): + call_data = call.data + call_data["type"] = TaskType.report + call_data["include_subprojects"] = True + + process_include_subprojects(call_data) + ret_params = {} + tasks = Task.get_many_with_join( + company=company_id, + query_dict=call_data, + allow_public=request.allow_public, + ret_params=ret_params, + ) + unprepare_from_saved(call, tasks) + + call.result.data = {"tasks": tasks, **ret_params} + + +def _get_task_metrics_from_request( + task_ids: Sequence[str], request: EventsRequest +) -> dict: + task_metrics = {} + for task in task_ids: + task_dict = {} + for mv in request.metrics: + task_dict[mv.metric] = mv.variants + task_metrics[task] = task_dict + + return task_metrics + + +@endpoint("reports.get_task_data", required_fields=[]) +def get_task_data(call: APICall, company_id, request: GetTasksDataRequest): + call_data = escape_execution_parameters(call) + process_include_subprojects(call_data) + + ret_params = {} + tasks = Task.get_many_with_join( + company=company_id, + query_dict=call_data, + query=_hidden_query(call_data), + allow_public=request.allow_public, + ret_params=ret_params, + ) + unprepare_from_saved(call, tasks) + res = {"tasks": tasks, **ret_params} + if not ( + request.debug_images + or request.plots + or request.scalar_metrics_iter_histogram + ): + return res + + task_ids = [task["id"] for task in tasks] + company, tasks_or_models = _get_task_or_model_index_company( + company_id, task_ids + ) + if request.debug_images: + result = event_bll.debug_images_iterator.get_task_events( + company_id=company, + task_metrics=_get_task_metrics_from_request(task_ids, request.debug_images), + iter_count=request.debug_images.iters, + ) + res["debug_images"] = [ + r.to_struct() for r in _get_metrics_response(result.metric_events) + ] + + if request.plots: + res["plots"] = _get_multitask_plots( + company=company, + tasks_or_models=tasks_or_models, + last_iters=request.plots.iters, + metrics=_get_metric_variants_from_request(request.plots.metrics), + )[0] + + if request.scalar_metrics_iter_histogram: + res[ + "scalar_metrics_iter_histogram" + ] = event_bll.metrics.compare_scalar_metrics_average_per_iter( + company_id=company_id, + tasks=tasks_or_models, + samples=request.scalar_metrics_iter_histogram.samples, + key=request.scalar_metrics_iter_histogram.key, + metric_variants=_get_metric_variants_from_request( + request.scalar_metrics_iter_histogram.metrics + ), + ) + + call.result.data = res + + +@endpoint("reports.move") +def move(call: APICall, company_id: str, request: MoveReportRequest): + if not (request.project or request.project_name): + raise errors.bad_request.MissingRequiredFields( + "project or project_name is required" + ) + + task = _assert_report( + company_id=company_id, + task_id=request.task, + only_fields=("project",), + ) + user_id = call.identity.user + project_name = request.project_name + if not project_name: + project = Project.get_for_writing( + company=company_id, id=request.project, _only=("name",) + ) + project_name = project.name + + project_id = _ensure_reports_project( + company=company_id, user=user_id, name=project_name + ) + + project_bll.move_under_project( + entity_cls=Task, + user=call.identity.user, + company=company_id, + ids=[request.task], + project=project_id, + ) + + _delete_reports_project_if_empty(task.project) + + return {"project_id": project_id} + + +@endpoint( + "reports.publish", response_data_model=UpdateResponse, +) +def publish(call: APICall, company_id, request: PublishReportRequest): + task = _assert_report( + company_id=company_id, task_id=request.task + ) + updates = ChangeStatusRequest( + task=task, + company=company_id, + new_status=TaskStatus.published, + force=True, + status_reason="", + status_message=request.message, + user_id=call.identity.user, + ).execute(published=datetime.utcnow()) + + call.result.data_model = UpdateResponse(**updates) + + +@endpoint("reports.archive") +def archive(call: APICall, company_id, request: ArchiveReportRequest): + task = _assert_report( + company_id=company_id, task_id=request.task + ) + archived = task.update( + status_message=request.message, + status_reason="", + add_to_set__system_tags=EntityVisibility.archived.value, + last_change=datetime.utcnow(), + last_changed_by=call.identity.user, + ) + + return {"archived": archived} + + +@endpoint("reports.unarchive") +def unarchive(call: APICall, company_id, request: ArchiveReportRequest): + task = _assert_report( + company_id=company_id, task_id=request.task + ) + unarchived = task.update( + status_message=request.message, + status_reason="", + pull__system_tags=EntityVisibility.archived.value, + last_change=datetime.utcnow(), + last_changed_by=call.identity.user, + ) + return {"unarchived": unarchived} + + +# @endpoint("reports.share") +# def share(call: APICall, company_id, request: ShareReportRequest): +# _assert_report( +# company_id=company_id, user_id=call.identity.user, task_id=request.task +# ) +# call.result.data = { +# "changed": task_bll.share_task( +# company_id=company_id, task_ids=[request.task], share=request.share +# ) +# } + + +@endpoint("reports.delete") +def delete(call: APICall, company_id, request: DeleteReportRequest): + task = _assert_report( + company_id=company_id, + task_id=request.task, + only_fields=("project",), + ) + if ( + task.status != TaskStatus.created + and EntityVisibility.archived.value not in task.system_tags + and not request.force + ): + raise errors.bad_request.TaskCannotBeDeleted( + "due to status, use force=True", + task=task.id, + expected=TaskStatus.created, + current=task.status, + ) + + task.delete() + _delete_reports_project_if_empty(task.project) + + call.result.data = {"deleted": 1} + + +@endpoint("reports.get_tags") +def get_tags(call: APICall, company_id: str, _): + tags = Task.objects(company=company_id, type=TaskType.report).distinct(field="tags") + call.result.data = sort_tags_response({"tags": tags}) diff --git a/apiserver/services/tasks.py b/apiserver/services/tasks.py index 377e85c..3992163 100644 --- a/apiserver/services/tasks.py +++ b/apiserver/services/tasks.py @@ -475,7 +475,9 @@ def _validate_and_get_task_from_call(call: APICall, **kwargs) -> Tuple[Task, dic field_does_not_exist_cls=errors.bad_request.ValidationError ): fields = prepare_create_fields(call, **kwargs) - task = task_bll.create(call, fields) + task = task_bll.create( + company=call.identity.company, user=call.identity.user, fields=fields + ) task_bll.validate(task) @@ -710,7 +712,11 @@ def edit(call: APICall, company_id, req_model: UpdateRequest): d.update(value) fields[key] = d - task_bll.validate(task_bll.create(call, fields)) + task_bll.validate( + task_bll.create( + company=call.identity.company, user=call.identity.user, fields=fields + ) + ) # make sure field names do not end in mongoengine comparison operators fixed_fields = { diff --git a/apiserver/tests/automated/__init__.py b/apiserver/tests/automated/__init__.py index 88ec01e..a0b3217 100644 --- a/apiserver/tests/automated/__init__.py +++ b/apiserver/tests/automated/__init__.py @@ -60,12 +60,12 @@ class TestService(TestCase, TestServiceInterface): def update_missing(target: dict, **update): target.update({k: v for k, v in update.items() if k not in target}) - def create_temp(self, service, *, client=None, delete_params=None, **kwargs) -> str: + def create_temp(self, service, *, client=None, delete_params=None, object_name="", **kwargs) -> str: return self._create_temp_helper( service=service, create_endpoint="create", delete_endpoint="delete", - object_name=service.rstrip("s"), + object_name=object_name or service.rstrip("s"), create_params=kwargs, client=client, delete_params=delete_params, diff --git a/apiserver/tests/automated/test_reports.py b/apiserver/tests/automated/test_reports.py new file mode 100644 index 0000000..c9fddba --- /dev/null +++ b/apiserver/tests/automated/test_reports.py @@ -0,0 +1,189 @@ +import re + +from boltons.iterutils import first + +from apiserver.apierrors import errors +from apiserver.es_factory import es_factory +from apiserver.tests.automated import TestService +from apiserver.utilities.dicts import nested_get + + +class TestReports(TestService): + def _delete_project(self, name): + existing_project = first( + self.api.projects.get_all_ex( + name=f"^{re.escape(name)}$", search_hidden=True + ).projects + ) + if existing_project: + self.api.projects.delete( + project=existing_project.id, force=True, delete_contents=True + ) + + def test_create_update_move(self): + task_name = "Rep1" + comment = "My report" + tags = ["hello"] + + # report creates a hidden task under hidden .reports subproject + self._delete_project(".reports") + task_id = self._temp_report(name=task_name, comment=comment, tags=tags) + task = self.api.tasks.get_all_ex(id=[task_id]).tasks[0] + self.assertEqual(task.name, task_name) + self.assertEqual(task.comment, comment) + self.assertEqual(set(task.tags), set(tags)) + self.assertEqual(task.type, "report") + self.assertEqual(set(task.system_tags), {"hidden", "reports"}) + projects = self.api.projects.get_all_ex(name=r"^\.reports$").projects + self.assertEqual(len(projects), 0) + project = self.api.projects.get_all_ex( + name=r"^\.reports$", search_hidden=True + ).projects[0] + self.assertEqual(project.id, task.project.id) + self.assertEqual(set(project.system_tags), {"hidden", "reports"}) + ret = self.api.reports.get_tags() + self.assertEqual(ret.tags, sorted(tags)) + + # update is working on draft reports + new_comment = "My new comment" + res = self.api.reports.update(task=task_id, comment=new_comment, tags=[]) + self.assertEqual(res.updated, 1) + task = self.api.tasks.get_all_ex(id=[task_id]).tasks[0] + self.assertEqual(task.name, task_name) + self.assertEqual(task.comment, new_comment) + self.assertEqual(task.tags, []) + ret = self.api.reports.get_tags() + self.assertEqual(ret.tags, []) + self.api.reports.publish(task=task_id) + with self.api.raises(errors.bad_request.InvalidTaskStatus): + self.api.reports.update(task=task_id, comment=comment) + + # move under another project autodeletes the empty project + new_project_name = "Reports Test" + self._delete_project(new_project_name) + task2_id = self._temp_report(name="Rep2") + new_project_id = self.api.reports.move( + task=task_id, project_name=new_project_name + ).project_id + new_project = self.api.projects.get_all_ex(id=[new_project_id]).projects[0] + self.assertEqual(new_project.name, f"{new_project_name}/.reports") + self.assertEqual(set(new_project.system_tags), {"hidden", "reports"}) + self.assertEqual(len(self.api.projects.get_all_ex(id=project.id).projects), 1) + self.api.reports.move(task=task2_id, project=new_project_id) + self.assertEqual(len(self.api.projects.get_all_ex(id=project.id).projects), 0) + tasks = self.api.tasks.get_all_ex( + project=new_project_id, search_hidden=True + ).tasks + self.assertTrue({task_id, task2_id}.issubset({t.id for t in tasks})) + + def test_reports_search(self): + report_task = self._temp_report(name="Rep1") + non_report_task = self._temp_task(name="hello") + res = self.api.reports.get_all_ex( + _any_={"pattern": "hello", "fields": ["name", "id", "tags", "report"]} + ).tasks + self.assertEqual(len(res), 0) + + self.api.reports.update(task=report_task, report="hello world") + res = self.api.reports.get_all_ex( + _any_={"pattern": "hello", "fields": ["name", "id", "tags", "report"]} + ).tasks + self.assertEqual(len(res), 1) + self.assertEqual(res[0].id, report_task) + + def test_reports_task_data(self): + report_task = self._temp_report(name="Rep1") + non_report_task = self._temp_task(name="hello") + debug_image_events = [ + self._create_task_event( + task=non_report_task, + type_="training_debug_image", + iteration=1, + metric=f"Metric_{m}", + variant=f"Variant_{v}", + url=f"{m}_{v}", + ) + for m in range(2) + for v in range(2) + ] + plot_events = [ + self._create_task_event( + task=non_report_task, + type_="plot", + iteration=1, + metric=f"Metric_{m}", + variant=f"Variant_{v}", + plot_str=f"Hello plot", + ) + for m in range(2) + for v in range(2) + ] + self.send_batch([*debug_image_events, *plot_events]) + + res = self.api.reports.get_task_data( + id=[non_report_task], + only_fields=["name"], + ) + self.assertEqual(len(res.tasks), 1) + self.assertEqual(res.tasks[0].id, non_report_task) + self.assertFalse(any(field in res for field in ("plots", "debug_images"))) + + res = self.api.reports.get_task_data( + id=[non_report_task], + only_fields=["name"], + debug_images={"metrics": []}, + plots={"metrics": [{"metric": "Metric_1"}]}, + ) + self.assertEqual(len(res.debug_images), 1) + task_events = res.debug_images[0] + self.assertEqual(task_events.task, non_report_task) + self.assertEqual(len(task_events.iterations), 1) + self.assertEqual(len(task_events.iterations[0].events), 4) + + self.assertEqual(len(res.plots), 1) + for m, v in (("Metric_1", "Variant_0"), ("Metric_1", "Variant_1")): + tasks = nested_get(res.plots, (m, v)) + self.assertEqual(len(tasks), 1) + task_plots = tasks[non_report_task] + self.assertEqual(len(task_plots), 1) + iter_plots = task_plots["1"] + self.assertEqual(iter_plots.name, "hello") + self.assertEqual(len(iter_plots.plots), 1) + ev = iter_plots.plots[0] + self.assertEqual(ev["metric"], m) + self.assertEqual(ev["variant"], v) + self.assertEqual(ev["task"], non_report_task) + self.assertEqual(ev["iter"], 1) + + @staticmethod + def _create_task_event(type_, task, iteration, **kwargs): + return { + "worker": "test", + "type": type_, + "task": task, + "iter": iteration, + "timestamp": kwargs.get("timestamp") or es_factory.get_timestamp_millis(), + **kwargs, + } + + def _temp_report(self, name, **kwargs): + return self.create_temp( + "reports", + name=name, + object_name="task", + delete_params={"force": True}, + **kwargs, + ) + + def _temp_task(self, name, **kwargs): + return self.create_temp( + "tasks", + name=name, + type="training", + delete_params={"force": True}, + **kwargs, + ) + + def send_batch(self, events): + _, data = self.api.send_batch("events.add_batch", events) + return data