diff --git a/apiserver/apimodels/login.py b/apiserver/apimodels/login.py index bb10e02..c8364dd 100644 --- a/apiserver/apimodels/login.py +++ b/apiserver/apimodels/login.py @@ -31,3 +31,4 @@ class GetSupportedModesResponse(Base): server_errors = EmbeddedField(ServerErrors) sso = DictField([str, type(None)]) sso_providers = ListField([dict]) + authenticated = BoolField(default=False) diff --git a/apiserver/bll/event/scalar_key.py b/apiserver/bll/event/scalar_key.py index b2b3e97..45b3448 100644 --- a/apiserver/bll/event/scalar_key.py +++ b/apiserver/bll/event/scalar_key.py @@ -4,8 +4,8 @@ Module for polymorphism over different types of X axes in scalar aggregations from abc import ABC, abstractmethod from enum import auto +from apiserver.utilities import extract_properties_to_lists from apiserver.utilities.stringenum import StringEnum -from apiserver.bll.util import extract_properties_to_lists from apiserver.config_repo import config log = config.logger(__file__) diff --git a/apiserver/bll/statistics/stats_reporter.py b/apiserver/bll/statistics/stats_reporter.py index 08477e6..e61e469 100644 --- a/apiserver/bll/statistics/stats_reporter.py +++ b/apiserver/bll/statistics/stats_reporter.py @@ -45,7 +45,7 @@ class StatisticsReporter: def start_reporter(cls): """ Periodically send statistics reports for companies who have opted in. - Note: in trains we usually have only a single company + Note: in clearml we usually have only a single company """ if not cls.supported: return diff --git a/apiserver/config/basic.py b/apiserver/config/basic.py index f6c69fc..478d2b3 100644 --- a/apiserver/config/basic.py +++ b/apiserver/config/basic.py @@ -6,8 +6,9 @@ from functools import reduce from os import getenv from os.path import expandvars from pathlib import Path -from typing import List, Any, TypeVar +from typing import List, Any, TypeVar, Sequence +from boltons.iterutils import first from pyhocon import ConfigTree, ConfigFactory from pyparsing import ( ParseFatalException, @@ -18,8 +19,8 @@ from pyparsing import ( from apiserver.utilities import json -EXTRA_CONFIG_PATHS = ("/opt/trains/config",) -EXTRA_CONFIG_PATH_OVERRIDE_VAR = "TRAINS_CONFIG_DIR" +EXTRA_CONFIG_PATHS = ("/opt/clearml/config",) +DEFAULT_PREFIXES = ("clearml", "trains") EXTRA_CONFIG_PATH_SEP = ":" if platform.system() != "Windows" else ";" @@ -30,7 +31,10 @@ class BasicConfig: default_config_dir = "default" def __init__( - self, folder: str = None, verbose: bool = True, prefix: str = "trains" + self, + folder: str = None, + verbose: bool = True, + prefix: Sequence[str] = DEFAULT_PREFIXES, ): folder = ( Path(folder) @@ -41,8 +45,16 @@ class BasicConfig: raise ValueError("Invalid configuration folder") self.verbose = verbose - self.prefix = prefix - self.extra_config_values_env_key_prefix = f"{self.prefix.upper()}__" + + self.extra_config_path_override_var = [ + f"{p.upper()}_CONFIG_DIR" for p in prefix + ] + + self.prefix = prefix[0] + self.extra_config_values_env_key_prefix = [ + f"{p.upper()}{self.extra_config_values_env_key_sep}" + for p in reversed(prefix) + ] self._paths = [folder, *self._get_paths()] self._config = self._reload() @@ -73,24 +85,24 @@ class BasicConfig: def _read_extra_env_config_values(self) -> ConfigTree: """ Loads extra configuration from environment-injected values """ result = ConfigTree() - prefix = self.extra_config_values_env_key_prefix - keys = sorted(k for k in os.environ if k.startswith(prefix)) - for key in keys: - path = ( - key[len(prefix) :] - .replace(self.extra_config_values_env_key_sep, ".") - .lower() - ) - result = ConfigTree.merge_configs( - result, ConfigFactory.parse_string(f"{path}: {os.environ[key]}") - ) + for prefix in self.extra_config_values_env_key_prefix: + keys = sorted(k for k in os.environ if k.startswith(prefix)) + for key in keys: + path = ( + key[len(prefix) :] + .replace(self.extra_config_values_env_key_sep, ".") + .lower() + ) + result = ConfigTree.merge_configs( + result, ConfigFactory.parse_string(f"{path}: {os.environ[key]}") + ) return result def _get_paths(self) -> List[Path]: default_paths = EXTRA_CONFIG_PATH_SEP.join(EXTRA_CONFIG_PATHS) - value = getenv(EXTRA_CONFIG_PATH_OVERRIDE_VAR, default_paths) + value = first(map(getenv, self.extra_config_path_override_var), default_paths) paths = [ Path(expandvars(v)).expanduser() for v in value.split(EXTRA_CONFIG_PATH_SEP) @@ -100,7 +112,7 @@ class BasicConfig: invalid = [path for path in paths if not path.is_dir()] if invalid: print( - f"WARNING: Invalid paths in {EXTRA_CONFIG_PATH_OVERRIDE_VAR} env var: {' '.join(map(str, invalid))}" + f"WARNING: Invalid paths in {self.extra_config_path_override_var} env var: {' '.join(map(str, invalid))}" ) return [path for path in paths if path.is_dir()] diff --git a/apiserver/config/info.py b/apiserver/config/info.py index 243686e..a3889b6 100644 --- a/apiserver/config/info.py +++ b/apiserver/config/info.py @@ -2,6 +2,8 @@ from functools import lru_cache from os import getenv from pathlib import Path +from boltons.iterutils import first + from apiserver.config_repo import config from apiserver.version import __version__ @@ -9,7 +11,9 @@ root = Path(__file__).parent.parent def _get(prop_name, env_suffix=None, default=""): - value = getenv(f"TRAINS_SERVER_{env_suffix or prop_name}") + suffix = env_suffix or prop_name + keys = [f"{p}_SERVER_{suffix}" for p in ("CLEARML", "TRAINS")] + value = first(map(getenv, keys)) if value: return value diff --git a/apiserver/database/__init__.py b/apiserver/database/__init__.py index b762ed6..8c5b4ea 100644 --- a/apiserver/database/__init__.py +++ b/apiserver/database/__init__.py @@ -17,11 +17,16 @@ log = config.logger("database") strict = config.get("apiserver.mongo.strict", True) OVERRIDE_HOST_ENV_KEY = ( + "CLEARML_MONGODB_SERVICE_HOST", "TRAINS_MONGODB_SERVICE_HOST", "MONGODB_SERVICE_HOST", "MONGODB_SERVICE_SERVICE_HOST", ) -OVERRIDE_PORT_ENV_KEY = ("TRAINS_MONGODB_SERVICE_PORT", "MONGODB_SERVICE_PORT") +OVERRIDE_PORT_ENV_KEY = ( + "CLEARML_MONGODB_SERVICE_PORT", + "TRAINS_MONGODB_SERVICE_PORT", + "MONGODB_SERVICE_PORT", +) class DatabaseEntry(models.Base): @@ -70,7 +75,9 @@ class DatabaseFactory: except ValidationError as ex: raise Exception("Invalid database entry `%s`: %s" % (key, ex.args[0])) if missing: - raise ValueError("Missing database configuration for %s" % ", ".join(missing)) + raise ValueError( + "Missing database configuration for %s" % ", ".join(missing) + ) @classmethod def get_entries(cls): diff --git a/apiserver/es_factory.py b/apiserver/es_factory.py index f48248d..d6522f8 100644 --- a/apiserver/es_factory.py +++ b/apiserver/es_factory.py @@ -9,11 +9,16 @@ from apiserver.config_repo import config log = config.logger(__file__) OVERRIDE_HOST_ENV_KEY = ( + "CLEARML_ELASTIC_SERVICE_HOST", "TRAINS_ELASTIC_SERVICE_HOST", "ELASTIC_SERVICE_HOST", "ELASTIC_SERVICE_SERVICE_HOST", ) -OVERRIDE_PORT_ENV_KEY = ("TRAINS_ELASTIC_SERVICE_PORT", "ELASTIC_SERVICE_PORT") +OVERRIDE_PORT_ENV_KEY = ( + "CLEARML_ELASTIC_SERVICE_PORT", + "TRAINS_ELASTIC_SERVICE_PORT", + "ELASTIC_SERVICE_PORT", +) OVERRIDE_HOST = first(filter(None, map(getenv, OVERRIDE_HOST_ENV_KEY))) if OVERRIDE_HOST: @@ -120,7 +125,9 @@ class ESFactory: @classmethod def get_es_timestamp_str(cls): now = datetime.utcnow() - return now.strftime("%Y-%m-%dT%H:%M:%S") + ".%03d" % (now.microsecond / 1000) + "Z" + return ( + now.strftime("%Y-%m-%dT%H:%M:%S") + ".%03d" % (now.microsecond / 1000) + "Z" + ) es_factory = ESFactory() diff --git a/apiserver/redis_manager.py b/apiserver/redis_manager.py index 17c2728..77a936d 100644 --- a/apiserver/redis_manager.py +++ b/apiserver/redis_manager.py @@ -11,8 +11,16 @@ from apiserver.config_repo import config log = config.logger(__file__) -OVERRIDE_HOST_ENV_KEY = ("TRAINS_REDIS_SERVICE_HOST", "REDIS_SERVICE_HOST") -OVERRIDE_PORT_ENV_KEY = ("TRAINS_REDIS_SERVICE_PORT", "REDIS_SERVICE_PORT") +OVERRIDE_HOST_ENV_KEY = ( + "CLEARML_REDIS_SERVICE_HOST", + "TRAINS_REDIS_SERVICE_HOST", + "REDIS_SERVICE_HOST", +) +OVERRIDE_PORT_ENV_KEY = ( + "CLEARML_REDIS_SERVICE_PORT", + "TRAINS_REDIS_SERVICE_PORT", + "REDIS_SERVICE_PORT", +) OVERRIDE_HOST = first(filter(None, map(getenv, OVERRIDE_HOST_ENV_KEY))) if OVERRIDE_HOST: diff --git a/apiserver/schema/services/login.conf b/apiserver/schema/services/login.conf index 41cd4b6..77a3cbb 100644 --- a/apiserver/schema/services/login.conf +++ b/apiserver/schema/services/login.conf @@ -85,6 +85,10 @@ supported_modes { } } } + authenticated { + description: "Is user authenticated" + type: boolean + } } } } diff --git a/apiserver/service_repo/__init__.py b/apiserver/service_repo/__init__.py index 6c811d5..e8d025e 100644 --- a/apiserver/service_repo/__init__.py +++ b/apiserver/service_repo/__init__.py @@ -1,6 +1,6 @@ from typing import Text, Sequence, Callable, Union, Type -from funcsigs import signature +from inspect import signature from jsonmodels import models from .apicall import APICall, APICallResult diff --git a/apiserver/service_repo/apicall.py b/apiserver/service_repo/apicall.py index b7c1ff2..e1401e8 100644 --- a/apiserver/service_repo/apicall.py +++ b/apiserver/service_repo/apicall.py @@ -5,6 +5,7 @@ from typing import Type, Optional, Union, Tuple import attr from jsonmodels import models +from requests.structures import CaseInsensitiveDict from six import string_types from apiserver import database @@ -313,6 +314,13 @@ class APICall(DataContainer): def HEADER_TRANSACTION(self): return self._transaction_headers[0] + _client_headers = _get_headers("Client") + """ Client """ + + @property + def HEADER_CLIENT(self): + return self._client_headers[0] + _worker_headers = _get_headers("Worker") """ Worker (machine) ID """ @@ -366,7 +374,7 @@ class APICall(DataContainer): assert isinstance(endpoint_version, PartialVersion), endpoint_version self._requested_endpoint_version = endpoint_version self._actual_endpoint_version = None - self._headers = {} + self._headers = CaseInsensitiveDict() self._kpis = {} self._log_api = True if headers: @@ -420,7 +428,7 @@ class APICall(DataContainer): :param header: Header name options (more than on supported, all will be cleared) """ for value in header if isinstance(header, (tuple, list)) else (header,): - self.headers.pop(value, None) + self._headers.pop(value, None) def set_header(self, header, value): """ @@ -514,7 +522,7 @@ class APICall(DataContainer): @property def headers(self): - return self._headers + return dict(self._headers.items()) @property def kpis(self): @@ -532,6 +540,10 @@ class APICall(DataContainer): def trx(self, value): self.set_header(self._transaction_headers, value) + @property + def client(self): + return self.get_header(self._client_headers) + @property def worker(self): return self.get_worker(default="") diff --git a/apiserver/services/debug.py b/apiserver/services/debug.py index afbc978..6c5d304 100644 --- a/apiserver/services/debug.py +++ b/apiserver/services/debug.py @@ -3,4 +3,4 @@ from apiserver.service_repo import APICall, endpoint @endpoint("debug.ping") def ping(call: APICall, _, __): - call.result.data = {"msg": "Because it trains cats and dogs"} + call.result.data = {"msg": "ClearML server"} diff --git a/apiserver/services/queues.py b/apiserver/services/queues.py index 7e43ad1..6d8f4cb 100644 --- a/apiserver/services/queues.py +++ b/apiserver/services/queues.py @@ -13,10 +13,10 @@ from apiserver.apimodels.queues import ( QueueMetrics, ) from apiserver.bll.queue import QueueBLL -from apiserver.bll.util import extract_properties_to_lists from apiserver.bll.workers import WorkerBLL from apiserver.service_repo import APICall, endpoint from apiserver.services.utils import conform_tag_fields, conform_output_tags, conform_tags +from apiserver.utilities import extract_properties_to_lists worker_bll = WorkerBLL() queue_bll = QueueBLL(worker_bll) diff --git a/apiserver/services/workers.py b/apiserver/services/workers.py index 45fff0c..0e0a87b 100644 --- a/apiserver/services/workers.py +++ b/apiserver/services/workers.py @@ -23,10 +23,10 @@ from apiserver.apimodels.workers import ( GetActivityReportResponse, ActivityReportSeries, ) -from apiserver.bll.util import extract_properties_to_lists from apiserver.bll.workers import WorkerBLL from apiserver.config_repo import config from apiserver.service_repo import APICall, endpoint +from apiserver.utilities import extract_properties_to_lists log = config.logger(__file__) diff --git a/apiserver/tests/api_client.py b/apiserver/tests/api_client.py index f4c1746..b67a733 100644 --- a/apiserver/tests/api_client.py +++ b/apiserver/tests/api_client.py @@ -154,7 +154,7 @@ class APIClient: self.http_session.mount("https://", adapter) if impersonated_user_id: - self.http_session.headers["X-Trains-Impersonate-As"] = impersonated_user_id + self.http_session.headers["X-ClearML-Impersonate-As"] = impersonated_user_id if not self.session_token: self.login() @@ -211,7 +211,7 @@ class APIClient: headers = {"Content-Type": "application/json"} headers.update(headers_overrides) if is_async: - headers["X-Trains-Async"] = "1" + headers["X-ClearML-Async"] = "1" if not isinstance(data, six.string_types): data = json.dumps(data) @@ -241,7 +241,7 @@ class APIClient: call_id = res.meta.call_id async_res_url = "%s/async.result?id=%s" % (self.base_url, call_id) async_res_headers = headers.copy() - async_res_headers.pop("X-Trains-Async") + async_res_headers.pop("X-ClearML-Async") while not got_result: log.info("Got 202. Checking async result for %s (%s)" % (url, call_id)) http_res = self.http_session.get( diff --git a/apiserver/updates.py b/apiserver/updates.py index f5e5e67..8d2ab06 100644 --- a/apiserver/updates.py +++ b/apiserver/updates.py @@ -37,12 +37,12 @@ class CheckUpdatesThread(Thread): @property def component_name(self) -> str: - return config.get("apiserver.check_for_updates.component_name", "trains-server") + return config.get("apiserver.check_for_updates.component_name", "clearml-server") def _check_new_version_available(self) -> Optional[_VersionResponse]: url = config.get( "apiserver.check_for_updates.url", - "https://updates.trains.allegro.ai/updates", + "https://updates.clear.ml/updates", ) uid = Settings.get_by_key("server.uuid") diff --git a/apiserver/utilities/__init__.py b/apiserver/utilities/__init__.py index 38fe151..c56376f 100644 --- a/apiserver/utilities/__init__.py +++ b/apiserver/utilities/__init__.py @@ -1,2 +1,28 @@ +from operator import itemgetter +from typing import Sequence, Optional, Callable, Tuple + + def strict_map(*args, **kwargs): return list(map(*args, **kwargs)) + + +def extract_properties_to_lists( + key_names: Sequence[str], + data: Sequence[dict], + extract_func: Optional[Callable[[dict], Tuple]] = None, +) -> dict: + """ + Given a list of dictionaries and names of dictionary keys + builds a dictionary with the requested keys and values lists + For the empty list return the dictionary of empty lists + :param key_names: names of the keys in the resulting dictionary + :param data: sequence of dictionaries to extract values from + :param extract_func: the optional callable that extracts properties + from a dictionary and put them in a tuple in the order corresponding to + key_names. If not specified then properties are extracted according to key_names + """ + if not data: + return {k: [] for k in key_names} + + value_sequences = zip(*map(extract_func or itemgetter(*key_names), data)) + return dict(zip(key_names, map(list, value_sequences))) diff --git a/docker/legacy/trains-server/docker-compose-unified.yml b/docker/legacy/trains-server/docker-compose-unified.yml deleted file mode 100644 index afeb5e8..0000000 --- a/docker/legacy/trains-server/docker-compose-unified.yml +++ /dev/null @@ -1,85 +0,0 @@ -version: "3.6" -services: - trainsserver: - command: - - -c - - "echo \"#!/bin/bash\" > /opt/trains/all.sh && echo \"/opt/trains/wrapper.sh webserver&\" >> /opt/trains/all.sh && echo \"/opt/trains/wrapper.sh fileserver&\" >> /opt/trains/all.sh && echo \"/opt/trains/wrapper.sh apiserver\" >> /opt/trains/all.sh && cat /opt/trains/all.sh && chmod +x /opt/trains/all.sh && /opt/trains/all.sh" - entrypoint: /bin/bash - container_name: trains-server - image: allegroai/trains:latest - ports: - - 8008:8008 - - 8080:80 - - 8081:8081 - restart: unless-stopped - volumes: - - /opt/trains/logs:/var/log/trains - - /opt/trains/data/fileserver:/mnt/fileserver - - /opt/trains/config:/opt/trains/config - - depends_on: - - redis - - mongo - - elasticsearch - environment: - TRAINS_ELASTIC_SERVICE_HOST: elasticsearch - TRAINS_ELASTIC_SERVICE_PORT: 9200 - TRAINS_MONGODB_SERVICE_HOST: mongo - TRAINS_MONGODB_SERVICE_PORT: 27017 - TRAINS_REDIS_SERVICE_HOST: redis - TRAINS_REDIS_SERVICE_PORT: 6379 - networks: - - backend - elasticsearch: - networks: - - backend - container_name: trains-elastic - environment: - ES_JAVA_OPTS: -Xms2g -Xmx2g - bootstrap.memory_lock: "true" - cluster.name: trains - cluster.routing.allocation.node_initial_primaries_recoveries: "500" - cluster.routing.allocation.disk.watermark.low: 10gb - cluster.routing.allocation.disk.watermark.high: 10gb - cluster.routing.allocation.disk.watermark.flood_stage: 10gb - discovery.zen.minimum_master_nodes: "1" - discovery.type: "single-node" - http.compression_level: "7" - node.ingest: "true" - node.name: trains - reindex.remote.whitelist: '*.*' - xpack.monitoring.enabled: "false" - xpack.security.enabled: "false" - ulimits: - memlock: - soft: -1 - hard: -1 - nofile: - soft: 65536 - hard: 65536 - image: docker.elastic.co/elasticsearch/elasticsearch:7.6.2 - restart: unless-stopped - volumes: - - /opt/trains/data/elastic_7:/usr/share/elasticsearch/data - mongo: - networks: - - backend - container_name: trains-mongo - image: mongo:3.6.5 - restart: unless-stopped - command: --setParameter internalQueryExecMaxBlockingSortBytes=196100200 - volumes: - - /opt/trains/data/mongo/db:/data/db - - /opt/trains/data/mongo/configdb:/data/configdb - redis: - networks: - - backend - container_name: trains-redis - image: redis:5.0 - restart: unless-stopped - volumes: - - /opt/trains/data/redis:/data - -networks: - backend: - driver: bridge diff --git a/docker/legacy/trains-server/docker-compose-win10.yml b/docker/legacy/trains-server/docker-compose-win10.yml index b44ddd3..c779638 100644 --- a/docker/legacy/trains-server/docker-compose-win10.yml +++ b/docker/legacy/trains-server/docker-compose-win10.yml @@ -39,9 +39,9 @@ services: bootstrap.memory_lock: "true" cluster.name: trains cluster.routing.allocation.node_initial_primaries_recoveries: "500" - cluster.routing.allocation.disk.watermark.low: 10gb - cluster.routing.allocation.disk.watermark.high: 10gb - cluster.routing.allocation.disk.watermark.flood_stage: 10gb + cluster.routing.allocation.disk.watermark.low: 500mb + cluster.routing.allocation.disk.watermark.high: 500mb + cluster.routing.allocation.disk.watermark.flood_stage: 500mb discovery.zen.minimum_master_nodes: "1" discovery.type: "single-node" http.compression_level: "7" diff --git a/docker/legacy/trains-server/docker-compose.yml b/docker/legacy/trains-server/docker-compose.yml index 0243a84..a309fe9 100644 --- a/docker/legacy/trains-server/docker-compose.yml +++ b/docker/legacy/trains-server/docker-compose.yml @@ -42,9 +42,9 @@ services: bootstrap.memory_lock: "true" cluster.name: trains cluster.routing.allocation.node_initial_primaries_recoveries: "500" - cluster.routing.allocation.disk.watermark.low: 10gb - cluster.routing.allocation.disk.watermark.high: 10gb - cluster.routing.allocation.disk.watermark.flood_stage: 10gb + cluster.routing.allocation.disk.watermark.low: 500mb + cluster.routing.allocation.disk.watermark.high: 500mb + cluster.routing.allocation.disk.watermark.flood_stage: 500mb discovery.zen.minimum_master_nodes: "1" discovery.type: "single-node" http.compression_level: "7" diff --git a/fileserver/config/basic.py b/fileserver/config/basic.py index 120e599..1df1f9d 100644 --- a/fileserver/config/basic.py +++ b/fileserver/config/basic.py @@ -5,6 +5,7 @@ from os import getenv from os.path import expandvars from pathlib import Path +from boltons.iterutils import first from pyhocon import ConfigTree, ConfigFactory from pyparsing import ( ParseFatalException, @@ -13,12 +14,10 @@ from pyparsing import ( ParseSyntaxException, ) -DEFAULT_EXTRA_CONFIG_PATH = "/opt/trains/config" -EXTRA_CONFIG_PATH_ENV_KEY = "TRAINS_CONFIG_DIR" +DEFAULT_EXTRA_CONFIG_PATH = "/opt/clearml/config" +PREFIXES = ("CLEARML", "TRAINS") EXTRA_CONFIG_PATH_SEP = ":" - EXTRA_CONFIG_VALUES_ENV_KEY_SEP = "__" -EXTRA_CONFIG_VALUES_ENV_KEY_PREFIX = f"TRAINS{EXTRA_CONFIG_VALUES_ENV_KEY_SEP}" class BasicConfig: @@ -29,7 +28,15 @@ class BasicConfig: if not self.folder.is_dir(): raise ValueError("Invalid configuration folder") - self.prefix = "trains" + self.extra_config_path_env_key = [ + f"{p.upper()}_CONFIG_DIR" for p in PREFIXES + ] + + self.prefix = PREFIXES[0] + self.extra_config_values_env_key_prefix = [ + f"{p.upper()}{EXTRA_CONFIG_VALUES_ENV_KEY_SEP}" + for p in reversed(PREFIXES) + ] self._load() @@ -50,24 +57,22 @@ class BasicConfig: path = ".".join((self.prefix, Path(name).stem)) return logging.getLogger(path) - @staticmethod - def _read_extra_env_config_values(): + def _read_extra_env_config_values(self): """ Loads extra configuration from environment-injected values """ result = ConfigTree() - prefix = EXTRA_CONFIG_VALUES_ENV_KEY_PREFIX - keys = sorted(k for k in os.environ if k.startswith(prefix)) - for key in keys: - path = key[len(prefix) :].replace(EXTRA_CONFIG_VALUES_ENV_KEY_SEP, ".").lower() - result = ConfigTree.merge_configs( - result, ConfigFactory.parse_string(f"{path}: {os.environ[key]}") - ) + for prefix in self.extra_config_values_env_key_prefix: + keys = sorted(k for k in os.environ if k.startswith(prefix)) + for key in keys: + path = key[len(prefix) :].replace(EXTRA_CONFIG_VALUES_ENV_KEY_SEP, ".").lower() + result = ConfigTree.merge_configs( + result, ConfigFactory.parse_string(f"{path}: {os.environ[key]}") + ) return result - @staticmethod - def _read_env_paths(key): - value = getenv(EXTRA_CONFIG_PATH_ENV_KEY, DEFAULT_EXTRA_CONFIG_PATH) + def _read_env_paths(self): + value = first(map(getenv, self.extra_config_path_env_key), DEFAULT_EXTRA_CONFIG_PATH) if value is None: return paths = [ @@ -79,11 +84,11 @@ class BasicConfig: if not path.is_dir() and str(path) != DEFAULT_EXTRA_CONFIG_PATH ] if invalid: - print(f"WARNING: Invalid paths in {key} env var: {' '.join(invalid)}") + print(f"WARNING: Invalid paths in {self.extra_config_path_env_key} env var: {' '.join(invalid)}") return [path for path in paths if path.is_dir()] def _load(self, verbose=True): - extra_config_paths = self._read_env_paths(EXTRA_CONFIG_PATH_ENV_KEY) or [] + extra_config_paths = self._read_env_paths() or [] extra_config_values = self._read_extra_env_config_values() configs = [ self._read_recursive(path, verbose=verbose) diff --git a/fileserver/config/default/logging.conf b/fileserver/config/default/logging.conf index 2c62e0d..edaa1a1 100644 --- a/fileserver/config/default/logging.conf +++ b/fileserver/config/default/logging.conf @@ -16,7 +16,7 @@ backupCount: 3 maxBytes: 10240000, class: "logging.handlers.RotatingFileHandler", - filename: "/var/log/trains/fileserver.log" + filename: "/var/log/clearml/fileserver.log" } } root {