Refactor es_factory into a separate class

This commit is contained in:
allegroai 2021-01-05 16:29:25 +02:00
parent f832ea565a
commit c7bbac73d0
8 changed files with 85 additions and 80 deletions

View File

@ -11,7 +11,7 @@ from mongoengine import Q
from nested_dict import nested_dict
from apiserver.database import utils as dbutils
from apiserver import es_factory
from apiserver.es_factory import es_factory
from apiserver.apierrors import errors
from apiserver.bll.event.debug_images_iterator import DebugImagesIterator
from apiserver.bll.event.event_metrics import EventMetrics, EventType

View File

@ -5,7 +5,7 @@ from typing import Callable, Sequence, Optional, Tuple
from elasticsearch import Elasticsearch
from apiserver import database
from apiserver import es_factory
from apiserver.es_factory import es_factory
from apiserver.apierrors import errors
from apiserver.bll.queue.queue_metrics import QueueMetrics
from apiserver.bll.workers import WorkerBLL

View File

@ -5,7 +5,7 @@ from typing import Sequence
import elasticsearch.helpers
from elasticsearch import Elasticsearch
from apiserver import es_factory
from apiserver.es_factory import es_factory
from apiserver.apierrors.errors import bad_request
from apiserver.bll.query import Builder as QueryBuilder
from apiserver.config import config

View File

@ -12,7 +12,7 @@ from mongoengine import Q
from six import string_types
import apiserver.database.utils as dbutils
from apiserver import es_factory
from apiserver.es_factory import es_factory
from apiserver.apierrors import errors
from apiserver.apimodels.tasks import Artifact as ApiArtifact
from apiserver.bll.organization import OrgBLL, Tags

View File

@ -5,7 +5,7 @@ from typing import Sequence, Set, Optional
import attr
import elasticsearch.helpers
from apiserver import es_factory
from apiserver.es_factory import es_factory
from apiserver.apierrors import APIError
from apiserver.apierrors.errors import bad_request, server_error
from apiserver.apimodels.workers import (

View File

@ -5,7 +5,7 @@ from typing import Type, Optional, Sequence, Any, Union
import urllib3.exceptions
from elasticsearch import Elasticsearch, exceptions
from apiserver import es_factory
from apiserver.es_factory import es_factory
from apiserver.config import config
from apiserver.elastic.apply_mappings import apply_mappings_to_cluster

View File

@ -42,7 +42,9 @@ class InvalidClusterConfiguration(Exception):
pass
def connect(cluster_name):
class ESFactory:
@classmethod
def connect(cls, cluster_name):
"""
Returns the es client for the cluster.
Connects to the cluster if did not connect previously
@ -52,7 +54,7 @@ def connect(cluster_name):
:raises InvalidClusterConfiguration: in case cluster config section misses needed properties
"""
if cluster_name not in _instances:
cluster_config = get_cluster_config(cluster_name)
cluster_config = cls.get_cluster_config(cluster_name)
hosts = cluster_config.get("hosts", None)
if not hosts:
raise InvalidClusterConfiguration(cluster_name)
@ -64,12 +66,12 @@ def connect(cluster_name):
return _instances[cluster_name]
def get_all_cluster_names():
@classmethod
def get_all_cluster_names(cls):
return list(config.get("hosts.elastic"))
def get_cluster_config(cluster_name):
@classmethod
def get_cluster_config(cls, cluster_name):
"""
Returns cluster config for the specified cluster path
:param cluster_name: Dot separated cluster path in the configuration file
@ -93,29 +95,32 @@ def get_cluster_config(cluster_name):
return cluster_config
def connect_all():
@classmethod
def connect_all(cls):
clusters = config.get("hosts.elastic").as_plain_ordered_dict()
for name in clusters:
connect(name)
cls.connect(name)
def instances():
@classmethod
def instances(cls):
return _instances
def timestamp_str_to_millis(ts_str):
@classmethod
def timestamp_str_to_millis(cls, ts_str):
epoch = datetime.utcfromtimestamp(0)
current_date = datetime.strptime(ts_str, "%Y-%m-%dT%H:%M:%S.%fZ")
return int((current_date - epoch).total_seconds() * 1000.0)
def get_timestamp_millis():
@classmethod
def get_timestamp_millis(cls):
now = datetime.utcnow()
epoch = datetime.utcfromtimestamp(0)
return int((now - epoch).total_seconds() * 1000.0)
def get_es_timestamp_str():
@classmethod
def get_es_timestamp_str(cls):
now = datetime.utcnow()
return now.strftime("%Y-%m-%dT%H:%M:%S") + ".%03d" % (now.microsecond / 1000) + "Z"
es_factory = ESFactory()

View File

@ -10,7 +10,7 @@ from typing import Sequence, Optional, Tuple
from boltons.iterutils import first
from apiserver import es_factory
from apiserver.es_factory import es_factory
from apiserver.apierrors.errors.bad_request import EventsNotAdded
from apiserver.tests.automated import TestService