mirror of
https://github.com/clearml/clearml-server
synced 2025-01-31 10:56:48 +00:00
86 lines
2.5 KiB
Python
86 lines
2.5 KiB
Python
from datetime import datetime
|
|
|
|
from elasticsearch import Elasticsearch, Transport
|
|
|
|
from config import config
|
|
|
|
log = config.logger(__file__)
|
|
|
|
_instances = {}
|
|
|
|
|
|
class MissingClusterConfiguration(Exception):
|
|
"""
|
|
Exception when cluster configuration is not found in config files
|
|
"""
|
|
pass
|
|
|
|
|
|
class InvalidClusterConfiguration(Exception):
|
|
"""
|
|
Exception when cluster configuration does not contain required properties
|
|
"""
|
|
pass
|
|
|
|
|
|
def connect(cluster_name):
|
|
"""
|
|
Returns the es client for the cluster.
|
|
Connects to the cluster if did not connect previously
|
|
:param cluster_name: Dot separated cluster path in the configuration file
|
|
:return: es client
|
|
:raises MissingClusterConfiguration: in case no config section is found for the cluster
|
|
:raises InvalidClusterConfiguration: in case cluster config section misses needed properties
|
|
"""
|
|
if cluster_name not in _instances:
|
|
cluster_config = _get_cluster_config(cluster_name)
|
|
hosts = cluster_config.get('hosts', None)
|
|
if not hosts:
|
|
raise InvalidClusterConfiguration(cluster_name)
|
|
args = cluster_config.get('args', {})
|
|
_instances[cluster_name] = Elasticsearch(hosts=hosts, transport_class=Transport, **args)
|
|
|
|
return _instances[cluster_name]
|
|
|
|
|
|
def _get_cluster_config(cluster_name):
|
|
"""
|
|
Returns cluster config for the specified cluster path
|
|
:param cluster_name: Dot separated cluster path in the configuration file
|
|
:return: config section for the cluster
|
|
:raises MissingClusterConfiguration: in case no config section is found for the cluster
|
|
"""
|
|
cluster_key = '.'.join(('hosts.elastic', cluster_name))
|
|
cluster_config = config.get(cluster_key, None)
|
|
if not cluster_config:
|
|
raise MissingClusterConfiguration(cluster_name)
|
|
|
|
return cluster_config
|
|
|
|
|
|
def connect_all():
|
|
clusters = config.get("hosts.elastic").as_plain_ordered_dict()
|
|
for name in clusters:
|
|
connect(name)
|
|
|
|
|
|
def instances():
|
|
return _instances
|
|
|
|
|
|
def timestamp_str_to_millis(ts_str):
|
|
epoch = datetime.utcfromtimestamp(0)
|
|
current_date = datetime.strptime(ts_str, "%Y-%m-%dT%H:%M:%S.%fZ")
|
|
return int((current_date - epoch).total_seconds() * 1000.0)
|
|
|
|
|
|
def get_timestamp_millis():
|
|
now = datetime.utcnow()
|
|
epoch = datetime.utcfromtimestamp(0)
|
|
return int((now - epoch).total_seconds() * 1000.0)
|
|
|
|
|
|
def get_es_timestamp_str():
|
|
now = datetime.utcnow()
|
|
return now.strftime("%Y-%m-%dT%H:%M:%S") + ".%03d" % (now.microsecond / 1000) + "Z"
|