ClearML/clearml-server
1
0
Fork 0
mirror of https://github.com/clearml/clearml-server synced 2025-06-26 23:15:47 +00:00
Code Issues Actions Packages Projects Releases Wiki Activity

Limit metrics and variants to avoid ES error

Browse Source
This commit is contained in:
allegroai 2020-02-04 18:18:26 +02:00
parent 0722b20c1c
commit 72be8bee19
1 changed files with 29 additions and 8 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

37
server/bll/event/event_bll.py
View File

@ -52,7 +52,9 @@ class EventBLL(object):
def __init__(self, events_es=None): def __init__(self, events_es=None):
self.es = events_es or es_factory.connect("events") self.es = events_es or es_factory.connect("events")
self._metrics = EventMetrics(self.es) self._metrics = EventMetrics(self.es)
self._skip_iteration_for_metric = set(config.get("services.events.ignore_iteration.metrics", [])) self._skip_iteration_for_metric = set(
config.get("services.events.ignore_iteration.metrics", [])
)
@property @property
def metrics(self) -> EventMetrics: def metrics(self) -> EventMetrics:
@ -124,7 +126,10 @@ class EventBLL(object):
if task_id is not None: if task_id is not None:
es_action["_routing"] = task_id es_action["_routing"] = task_id
task_ids.add(task_id) task_ids.add(task_id)
if iter is not None and event.get("metric") not in self._skip_iteration_for_metric: if (
iter is not None
and event.get("metric") not in self._skip_iteration_for_metric
):
task_iteration[task_id] = max(iter, task_iteration[task_id]) task_iteration[task_id] = max(iter, task_iteration[task_id])
if event_type == EventType.metrics_scalar.value: if event_type == EventType.metrics_scalar.value:
@ -297,10 +302,16 @@ class EventBLL(object):
"size": 0, "size": 0,
"aggs": { "aggs": {
"metrics": { "metrics": {
"terms": {"field": "metric"}, "terms": {
"field": "metric",
"size": EventMetrics.MAX_METRICS_COUNT,
},
"aggs": { "aggs": {
"variants": { "variants": {
"terms": {"field": "variant"}, "terms": {
"field": "variant",
"size": EventMetrics.MAX_VARIANTS_COUNT,
},
"aggs": { "aggs": {
"iters": { "iters": {
"terms": { "terms": {
@ -499,8 +510,18 @@ class EventBLL(object):
"size": 0, "size": 0,
"aggs": { "aggs": {
"metrics": { "metrics": {
"terms": {"field": "metric", "size": 200}, "terms": {
"aggs": {"variants": {"terms": {"field": "variant", "size": 200}}}, "field": "metric",
"size": EventMetrics.MAX_METRICS_COUNT,
},
"aggs": {
"variants": {
"terms": {
"field": "variant",
"size": EventMetrics.MAX_VARIANTS_COUNT,
}
}
},
} }
}, },
"query": {"bool": {"must": [{"term": {"task": task_id}}]}}, "query": {"bool": {"must": [{"term": {"task": task_id}}]}},
@ -540,14 +561,14 @@ class EventBLL(object):
"metrics": { "metrics": {
"terms": { "terms": {
"field": "metric", "field": "metric",
"size": 1000, "size": EventMetrics.MAX_METRICS_COUNT,
"order": {"_term": "asc"}, "order": {"_term": "asc"},
}, },
"aggs": { "aggs": {
"variants": { "variants": {
"terms": { "terms": {
"field": "variant", "field": "variant",
"size": 1000, "size": EventMetrics.MAX_VARIANTS_COUNT,
"order": {"_term": "asc"}, "order": {"_term": "asc"},
}, },
"aggs": { "aggs": {