ClearML/clearml-server
1
0
Fork 0
mirror of https://github.com/clearml/clearml-server synced 2025-01-31 02:46:53 +00:00
Code Issues Actions Packages Projects Releases Wiki Activity

Add finer control for mongo/elastic/redis host configuration

Browse Source
This commit is contained in:
allegroai 2019-11-09 21:29:23 +02:00
parent 17d94b26c3
commit 8e7e234161
4 changed files with 42 additions and 22 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

13
server/database/__init__.py
View File

@ -1,5 +1,6 @@
from os import getenv
from boltons.iterutils import first
from furl import furl
from jsonmodels import models
from jsonmodels.errors import ValidationError
@ -11,14 +12,16 @@ from config import config
from .defs import Database
from .utils import get_items
from boltons.iterutils import first
log = config.logger("database")
strict = config.get("apiserver.mongo.strict", True)
OVERRIDE_HOST_ENV_KEY = ("MONGODB_SERVICE_HOST", "MONGODB_SERVICE_SERVICE_HOST")
OVERRIDE_PORT_ENV_KEY = "MONGODB_SERVICE_PORT"
OVERRIDE_HOST_ENV_KEY = (
"TRAINS_MONGODB_SERVICE_HOST",
"MONGODB_SERVICE_HOST",
"MONGODB_SERVICE_SERVICE_HOST",
)
OVERRIDE_PORT_ENV_KEY = ("TRAINS_MONGODB_SERVICE_PORT", "MONGODB_SERVICE_PORT")
_entries = []
@ -41,7 +44,7 @@ def initialize():
if override_hostname:
log.info(f"Using override mongodb host {override_hostname}")
override_port = getenv(OVERRIDE_PORT_ENV_KEY)
override_port = first(map(getenv, OVERRIDE_PORT_ENV_KEY), None)
if override_port:
log.info(f"Using override mongodb port {override_port}")

27
server/es_factory.py
View File

@ -1,20 +1,25 @@
from datetime import datetime
from os import getenv
from boltons.iterutils import first
from elasticsearch import Elasticsearch, Transport
from config import config
log = config.logger(__file__)
OVERRIDE_HOST_ENV_KEY = ("ELASTIC_SERVICE_HOST", "ELASTIC_SERVICE_SERVICE_HOST")
OVERRIDE_PORT_ENV_KEY = "ELASTIC_SERVICE_PORT"
OVERRIDE_HOST_ENV_KEY = (
"TRAINS_ELASTIC_SERVICE_HOST",
"ELASTIC_SERVICE_HOST",
"ELASTIC_SERVICE_SERVICE_HOST",
)
OVERRIDE_PORT_ENV_KEY = ("TRAINS_ELASTIC_SERVICE_PORT", "ELASTIC_SERVICE_PORT")
OVERRIDE_HOST = next(filter(None, map(getenv, OVERRIDE_HOST_ENV_KEY)), None)
OVERRIDE_HOST = first(filter(None, map(getenv, OVERRIDE_HOST_ENV_KEY)))
if OVERRIDE_HOST:
log.info(f"Using override elastic host {OVERRIDE_HOST}")
OVERRIDE_PORT = getenv(OVERRIDE_PORT_ENV_KEY)
OVERRIDE_PORT = first(filter(None, map(getenv, OVERRIDE_PORT_ENV_KEY)))
if OVERRIDE_PORT:
log.info(f"Using override elastic port {OVERRIDE_PORT}")
@ -25,6 +30,7 @@ class MissingClusterConfiguration(Exception):
"""
Exception when cluster configuration is not found in config files
"""
pass
@ -32,6 +38,7 @@ class InvalidClusterConfiguration(Exception):
"""
Exception when cluster configuration does not contain required properties
"""
pass
@ -46,12 +53,14 @@ def connect(cluster_name):
"""
if cluster_name not in _instances:
cluster_config = get_cluster_config(cluster_name)
hosts = cluster_config.get('hosts', None)
hosts = cluster_config.get("hosts", None)
if not hosts:
raise InvalidClusterConfiguration(cluster_name)
args = cluster_config.get('args', {})
_instances[cluster_name] = Elasticsearch(hosts=hosts, transport_class=Transport, **args)
args = cluster_config.get("args", {})
_instances[cluster_name] = Elasticsearch(
hosts=hosts, transport_class=Transport, **args
)
return _instances[cluster_name]
@ -63,13 +72,13 @@ def get_cluster_config(cluster_name):
:return: config section for the cluster
:raises MissingClusterConfiguration: in case no config section is found for the cluster
"""
cluster_key = '.'.join(('hosts.elastic', cluster_name))
cluster_key = ".".join(("hosts.elastic", cluster_name))
cluster_config = config.get(cluster_key, None)
if not cluster_config:
raise MissingClusterConfiguration(cluster_name)
def set_host_prop(key, value):
for host in cluster_config.get('hosts', []):
for host in cluster_config.get("hosts", []):
host[key] = value
if OVERRIDE_HOST:

10
server/service_repo/apicall.py
View File

@ -255,6 +255,7 @@ class MissingIdentity(Exception):
class APICall(DataContainer):
HEADER_AUTHORIZATION = "Authorization"
HEADER_REAL_IP = "X-Real-IP"
HEADER_FORWARDED_FOR = "X-Forwarded-For"
""" Standard headers """
_transaction_headers = ("X-Trains-Trx",)
@ -382,8 +383,13 @@ class APICall(DataContainer):
@property
def real_ip(self):
real_ip = self.get_header(self.HEADER_REAL_IP)
return real_ip or self._remote_addr or "untrackable"
""" Obtain visitor's IP address """
return (
self.get_header(self.HEADER_FORWARDED_FOR)
or self.get_header(self.HEADER_REAL_IP)
or self._remote_addr
or "untrackable"
)
@property
def failed(self):

14
server/service_repo/redis_manager.py
View File

@ -2,21 +2,23 @@ import threading
from os import getenv
from time import sleep
from apierrors.errors.server_error import ConfigError, GeneralError
from config import config
from boltons.iterutils import first
from redis import StrictRedis
from redis.sentinel import Sentinel, SentinelConnectionPool
from apierrors.errors.server_error import ConfigError, GeneralError
from config import config
log = config.logger(__file__)
OVERRIDE_HOST_ENV_KEY = "REDIS_SERVICE_HOST"
OVERRIDE_PORT_ENV_KEY = "REDIS_SERVICE_PORT"
OVERRIDE_HOST_ENV_KEY = ("TRAINS_REDIS_SERVICE_HOST", "REDIS_SERVICE_HOST")
OVERRIDE_PORT_ENV_KEY = ("TRAINS_REDIS_SERVICE_PORT", "REDIS_SERVICE_PORT")
OVERRIDE_HOST = getenv(OVERRIDE_HOST_ENV_KEY)
OVERRIDE_HOST = first(filter(None, map(getenv, OVERRIDE_HOST_ENV_KEY)))
if OVERRIDE_HOST:
log.info(f"Using override redis host {OVERRIDE_HOST}")
OVERRIDE_PORT = getenv(OVERRIDE_PORT_ENV_KEY)
OVERRIDE_PORT = first(filter(None, map(getenv, OVERRIDE_PORT_ENV_KEY)))
if OVERRIDE_PORT:
log.info(f"Using override redis port {OVERRIDE_PORT}")