From 8e7e234161cb589fba88a89385d583c04254896f Mon Sep 17 00:00:00 2001 From: allegroai <> Date: Sat, 9 Nov 2019 21:29:23 +0200 Subject: [PATCH] Add finer control for mongo/elastic/redis host configuration --- server/database/__init__.py | 13 ++++++++----- server/es_factory.py | 27 ++++++++++++++++++--------- server/service_repo/apicall.py | 10 ++++++++-- server/service_repo/redis_manager.py | 14 ++++++++------ 4 files changed, 42 insertions(+), 22 deletions(-) diff --git a/server/database/__init__.py b/server/database/__init__.py index 5b370ef..8c666a3 100644 --- a/server/database/__init__.py +++ b/server/database/__init__.py @@ -1,5 +1,6 @@ from os import getenv +from boltons.iterutils import first from furl import furl from jsonmodels import models from jsonmodels.errors import ValidationError @@ -11,14 +12,16 @@ from config import config from .defs import Database from .utils import get_items -from boltons.iterutils import first - log = config.logger("database") strict = config.get("apiserver.mongo.strict", True) -OVERRIDE_HOST_ENV_KEY = ("MONGODB_SERVICE_HOST", "MONGODB_SERVICE_SERVICE_HOST") -OVERRIDE_PORT_ENV_KEY = "MONGODB_SERVICE_PORT" +OVERRIDE_HOST_ENV_KEY = ( + "TRAINS_MONGODB_SERVICE_HOST", + "MONGODB_SERVICE_HOST", + "MONGODB_SERVICE_SERVICE_HOST", +) +OVERRIDE_PORT_ENV_KEY = ("TRAINS_MONGODB_SERVICE_PORT", "MONGODB_SERVICE_PORT") _entries = [] @@ -41,7 +44,7 @@ def initialize(): if override_hostname: log.info(f"Using override mongodb host {override_hostname}") - override_port = getenv(OVERRIDE_PORT_ENV_KEY) + override_port = first(map(getenv, OVERRIDE_PORT_ENV_KEY), None) if override_port: log.info(f"Using override mongodb port {override_port}") diff --git a/server/es_factory.py b/server/es_factory.py index 4b195bd..65573b1 100644 --- a/server/es_factory.py +++ b/server/es_factory.py @@ -1,20 +1,25 @@ from datetime import datetime from os import getenv +from boltons.iterutils import first from elasticsearch import Elasticsearch, Transport from config import config log = config.logger(__file__) -OVERRIDE_HOST_ENV_KEY = ("ELASTIC_SERVICE_HOST", "ELASTIC_SERVICE_SERVICE_HOST") -OVERRIDE_PORT_ENV_KEY = "ELASTIC_SERVICE_PORT" +OVERRIDE_HOST_ENV_KEY = ( + "TRAINS_ELASTIC_SERVICE_HOST", + "ELASTIC_SERVICE_HOST", + "ELASTIC_SERVICE_SERVICE_HOST", +) +OVERRIDE_PORT_ENV_KEY = ("TRAINS_ELASTIC_SERVICE_PORT", "ELASTIC_SERVICE_PORT") -OVERRIDE_HOST = next(filter(None, map(getenv, OVERRIDE_HOST_ENV_KEY)), None) +OVERRIDE_HOST = first(filter(None, map(getenv, OVERRIDE_HOST_ENV_KEY))) if OVERRIDE_HOST: log.info(f"Using override elastic host {OVERRIDE_HOST}") -OVERRIDE_PORT = getenv(OVERRIDE_PORT_ENV_KEY) +OVERRIDE_PORT = first(filter(None, map(getenv, OVERRIDE_PORT_ENV_KEY))) if OVERRIDE_PORT: log.info(f"Using override elastic port {OVERRIDE_PORT}") @@ -25,6 +30,7 @@ class MissingClusterConfiguration(Exception): """ Exception when cluster configuration is not found in config files """ + pass @@ -32,6 +38,7 @@ class InvalidClusterConfiguration(Exception): """ Exception when cluster configuration does not contain required properties """ + pass @@ -46,12 +53,14 @@ def connect(cluster_name): """ if cluster_name not in _instances: cluster_config = get_cluster_config(cluster_name) - hosts = cluster_config.get('hosts', None) + hosts = cluster_config.get("hosts", None) if not hosts: raise InvalidClusterConfiguration(cluster_name) - args = cluster_config.get('args', {}) - _instances[cluster_name] = Elasticsearch(hosts=hosts, transport_class=Transport, **args) + args = cluster_config.get("args", {}) + _instances[cluster_name] = Elasticsearch( + hosts=hosts, transport_class=Transport, **args + ) return _instances[cluster_name] @@ -63,13 +72,13 @@ def get_cluster_config(cluster_name): :return: config section for the cluster :raises MissingClusterConfiguration: in case no config section is found for the cluster """ - cluster_key = '.'.join(('hosts.elastic', cluster_name)) + cluster_key = ".".join(("hosts.elastic", cluster_name)) cluster_config = config.get(cluster_key, None) if not cluster_config: raise MissingClusterConfiguration(cluster_name) def set_host_prop(key, value): - for host in cluster_config.get('hosts', []): + for host in cluster_config.get("hosts", []): host[key] = value if OVERRIDE_HOST: diff --git a/server/service_repo/apicall.py b/server/service_repo/apicall.py index 25ae537..f125db5 100644 --- a/server/service_repo/apicall.py +++ b/server/service_repo/apicall.py @@ -255,6 +255,7 @@ class MissingIdentity(Exception): class APICall(DataContainer): HEADER_AUTHORIZATION = "Authorization" HEADER_REAL_IP = "X-Real-IP" + HEADER_FORWARDED_FOR = "X-Forwarded-For" """ Standard headers """ _transaction_headers = ("X-Trains-Trx",) @@ -382,8 +383,13 @@ class APICall(DataContainer): @property def real_ip(self): - real_ip = self.get_header(self.HEADER_REAL_IP) - return real_ip or self._remote_addr or "untrackable" + """ Obtain visitor's IP address """ + return ( + self.get_header(self.HEADER_FORWARDED_FOR) + or self.get_header(self.HEADER_REAL_IP) + or self._remote_addr + or "untrackable" + ) @property def failed(self): diff --git a/server/service_repo/redis_manager.py b/server/service_repo/redis_manager.py index bae8e42..8b85c68 100644 --- a/server/service_repo/redis_manager.py +++ b/server/service_repo/redis_manager.py @@ -2,21 +2,23 @@ import threading from os import getenv from time import sleep -from apierrors.errors.server_error import ConfigError, GeneralError -from config import config +from boltons.iterutils import first from redis import StrictRedis from redis.sentinel import Sentinel, SentinelConnectionPool +from apierrors.errors.server_error import ConfigError, GeneralError +from config import config + log = config.logger(__file__) -OVERRIDE_HOST_ENV_KEY = "REDIS_SERVICE_HOST" -OVERRIDE_PORT_ENV_KEY = "REDIS_SERVICE_PORT" +OVERRIDE_HOST_ENV_KEY = ("TRAINS_REDIS_SERVICE_HOST", "REDIS_SERVICE_HOST") +OVERRIDE_PORT_ENV_KEY = ("TRAINS_REDIS_SERVICE_PORT", "REDIS_SERVICE_PORT") -OVERRIDE_HOST = getenv(OVERRIDE_HOST_ENV_KEY) +OVERRIDE_HOST = first(filter(None, map(getenv, OVERRIDE_HOST_ENV_KEY))) if OVERRIDE_HOST: log.info(f"Using override redis host {OVERRIDE_HOST}") -OVERRIDE_PORT = getenv(OVERRIDE_PORT_ENV_KEY) +OVERRIDE_PORT = first(filter(None, map(getenv, OVERRIDE_PORT_ENV_KEY))) if OVERRIDE_PORT: log.info(f"Using override redis port {OVERRIDE_PORT}")