ClearML/clearml-server
1
0
Fork 0
mirror of https://github.com/clearml/clearml-server synced 2025-06-26 23:15:47 +00:00
Code Issues Actions Packages Projects Releases Wiki Activity

Avoid updating experiment last iteration for metric events related to machine/gpu monitoring

Browse Source
This commit is contained in:
allegroai 2019-12-21 18:14:13 +02:00
parent 53296e8891
commit 52529d3c55
2 changed files with 7 additions and 3 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
server/bll/event/event_bll.py
View File

@ -16,6 +16,7 @@ import es_factory
from apierrors import errors from apierrors import errors
from bll.event.event_metrics import EventMetrics from bll.event.event_metrics import EventMetrics
from bll.task import TaskBLL from bll.task import TaskBLL
from config import config
from database.errors import translate_errors_context from database.errors import translate_errors_context
from database.model.task.task import Task, TaskStatus from database.model.task.task import Task, TaskStatus
from timing_context import TimingContext from timing_context import TimingContext
@ -50,6 +51,7 @@ class EventBLL(object):
def __init__(self, events_es=None): def __init__(self, events_es=None):
self.es = events_es or es_factory.connect("events") self.es = events_es or es_factory.connect("events")
self._metrics = EventMetrics(self.es) self._metrics = EventMetrics(self.es)
self._skip_iteration_for_metric = set(config.get("services.events.ignore_iteration.metrics", []))
@property @property
def metrics(self) -> EventMetrics: def metrics(self) -> EventMetrics:
@ -121,7 +123,7 @@ class EventBLL(object):
if task_id is not None: if task_id is not None:
es_action["_routing"] = task_id es_action["_routing"] = task_id
task_ids.add(task_id) task_ids.add(task_id)
if iter is not None: if iter is not None and event.get("metric") not in self._skip_iteration_for_metric:
task_iteration[task_id] = max(iter, task_iteration[task_id]) task_iteration[task_id] = max(iter, task_iteration[task_id])
if event_type == EventType.metrics_scalar.value: if event_type == EventType.metrics_scalar.value:

6
server/config/default/services/events.conf
View File

@ -1,3 +1,5 @@
{ es_index_prefix: "events"
es_index_prefix:"events"
ignore_iteration {
metrics: [":monitor:machine", ":monitor:gpu"]
} }