ClearML/clearml-server
1
0
Fork 0
mirror of https://github.com/clearml/clearml-server synced 2025-03-03 10:43:10 +00:00
Code Issues Actions Packages Projects Releases Wiki Activity

Task reports can now return single value metrics

Browse Source
This commit is contained in:
allegroai 2023-05-25 19:18:24 +03:00
parent 58465fbc17
commit e99817b28b
7 changed files with 115 additions and 41 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

9
apiserver/apimodels/reports.py
View File

@ -61,10 +61,17 @@ class ScalarMetricsIterHistogram(HistogramRequestBase):
metrics: Sequence[MetricVariants] = ListField(items_types=MetricVariants)
class SingleValueMetrics(Base):
pass
class GetTasksDataRequest(Base):
debug_images: EventsRequest = EmbeddedField(EventsRequest)
plots: EventsRequest = EmbeddedField(EventsRequest)
scalar_metrics_iter_histogram: ScalarMetricsIterHistogram = EmbeddedField(ScalarMetricsIterHistogram)
scalar_metrics_iter_histogram: ScalarMetricsIterHistogram = EmbeddedField(
ScalarMetricsIterHistogram
)
single_value_metrics: SingleValueMetrics = EmbeddedField(SingleValueMetrics)
allow_public = BoolField(default=True)
model_events: bool = BoolField(default=False)

31
apiserver/schema/services/_events_common.conf
View File

@ -103,4 +103,35 @@ plots_response {
items {"$ref": "#/definitions/plots_response_task_metrics"}
}
}
}
single_value_task_metrics {
type: object
properties {
task {
type: string
description: Task ID
}
values {
type: array
items {
type: object
properties {
metric { type: string }
variant { type: string}
value { type: number }
timestamp { type: number }
}
}
}
}
}
single_value_metrics_response {
type: object
properties {
tasks {
description: Single value metrics grouped by task
type: array
items {"$ref": "#/definitions/single_value_task_metrics"}
}
}
}

31
apiserver/schema/services/events.conf
View File

@ -1353,36 +1353,7 @@ get_task_single_value_metrics {
}
}
}
response {
type: object
properties {
tasks {
description: Single value metrics grouped by task
type: array
items {
type: object
properties {
task {
type: string
description: Task ID
}
values {
type: array
items {
type: object
properties {
metric { type: string }
variant { type: string}
value { type: number }
timestamp { type: number }
}
}
}
}
}
}
}
}
response {"$ref": "#/definitions/single_value_metrics_response"}
}
"2.22": ${get_task_single_value_metrics."2.20"} {
request.properties.model_events {

20
apiserver/schema/services/reports.conf
View File

@ -569,10 +569,22 @@ get_task_data {
}
}
"999.0": ${get_task_data."2.23"} {
request.properties.model_events {
type: boolean
description: If set then the retrieving model events. Otherwise task events
default: false
request.properties {
model_events {
type: boolean
description: If set then the retrieving model events. Otherwise task events
default: false
}
single_value_metrics {
type: object
description: If passed then task single value metrics are returned
additonalProperties: false
}
}
response.properties.single_value_metrics {
type: array
description: Single value metrics grouped by task
items {"$ref": "#/definitions/single_value_task_metrics"}
}
}
}

10
apiserver/services/events.py
View File

@ -508,6 +508,12 @@ def multi_task_scalar_metrics_iter_histogram(
)
def _get_single_value_metrics_response(
value_metrics: Mapping[str, dict]
) -> Sequence[dict]:
return [{"task": task, "values": values} for task, values in value_metrics.items()]
@endpoint("events.get_task_single_value_metrics")
def get_task_single_value_metrics(
call, company_id: str, request: SingleValueMetricsRequest
@ -517,9 +523,7 @@ def get_task_single_value_metrics(
company_id, request.tasks, request.model_events
),
)
call.result.data = dict(
tasks=[{"task": task, "values": values} for task, values in res.items()]
)
call.result.data = dict(tasks=_get_single_value_metrics_response(res))
@endpoint("events.get_multi_task_plots", required_fields=["tasks"])

11
apiserver/services/reports.py
View File

@ -33,6 +33,7 @@ from apiserver.services.events import (
_get_metrics_response,
_get_metric_variants_from_request,
_get_multitask_plots,
_get_single_value_metrics_response,
)
from apiserver.services.tasks import (
escape_execution_parameters,
@ -221,7 +222,10 @@ def get_task_data(call: APICall, company_id, request: GetTasksDataRequest):
conform_data(call, tasks)
res = {"tasks": tasks, **ret_params}
if not (
request.debug_images or request.plots or request.scalar_metrics_iter_histogram
request.debug_images
or request.plots
or request.scalar_metrics_iter_histogram
or request.single_value_metrics
):
return res
@ -260,6 +264,11 @@ def get_task_data(call: APICall, company_id, request: GetTasksDataRequest):
),
)
if request.single_value_metrics:
res["single_value_metrics"] = _get_single_value_metrics_response(
event_bll.metrics.get_task_single_value_metrics(companies=companies)
)
call.result.data = res

44
apiserver/tests/automated/test_reports.py
View File

@ -146,20 +146,46 @@ class TestReports(TestService):
for m in range(2)
for v in range(2)
]
self.send_batch([*debug_image_events, *plot_events])
scalar_events = [
self._create_task_event(
task=non_report_task,
type_="training_stats_scalar",
iteration=iter_,
metric=f"Metric_{m}",
variant=f"Variant_{v}",
value=m * v,
**event_args,
)
for m in range(2)
for v in range(2)
for iter_ in (1, -(2 ** 31))
]
self.send_batch([*debug_image_events, *plot_events, *scalar_events])
res = self.api.reports.get_task_data(
id=[non_report_task], only_fields=["name"], model_events=model_events
)
self.assertEqual(len(res.tasks), 1)
self.assertEqual(res.tasks[0].id, non_report_task)
self.assertFalse(any(field in res for field in ("plots", "debug_images")))
self.assertFalse(
any(
field in res
for field in (
"plots",
"debug_images",
"scalar_metrics_iter_histogram",
"single_value_metrics",
)
)
)
res = self.api.reports.get_task_data(
id=[non_report_task],
only_fields=["name"],
debug_images={"metrics": []},
plots={"metrics": [{"metric": "Metric_1"}]},
scalar_metrics_iter_histogram={},
single_value_metrics={},
model_events=model_events,
)
self.assertEqual(len(res.debug_images), 1)
@ -168,6 +194,20 @@ class TestReports(TestService):
self.assertEqual(len(task_events.iterations), 1)
self.assertEqual(len(task_events.iterations[0].events), 4)
self.assertEqual(len(res.single_value_metrics), 1)
task_metrics = res.single_value_metrics[0]
self.assertEqual(task_metrics.task, non_report_task)
self.assertEqual(
{(v["metric"], v["variant"]) for v in task_metrics["values"]},
{(f"Metric_{x}", f"Variant_{y}") for x in range(2) for y in range(2)},
)
self.assertEqual(len(task_events.iterations[0].events), 4)
for m in ("Metric_0", "Metric_1"):
for v in ("Variant_0", "Variant_1"):
tasks = nested_get(res.scalar_metrics_iter_histogram, (m, v))
self.assertEqual(list(tasks.keys()), [non_report_task])
self.assertEqual(len(res.plots), 1)
for m, v in (("Metric_1", "Variant_0"), ("Metric_1", "Variant_1")):
tasks = nested_get(res.plots, (m, v))