mirror of
https://github.com/clearml/clearml-server
synced 2025-01-31 19:06:55 +00:00
144 lines
4.6 KiB
Python
144 lines
4.6 KiB
Python
import logging
|
|
import os
|
|
from functools import reduce
|
|
from os import getenv
|
|
from os.path import expandvars
|
|
from pathlib import Path
|
|
|
|
from pyhocon import ConfigTree, ConfigFactory
|
|
from pyparsing import (
|
|
ParseFatalException,
|
|
ParseException,
|
|
RecursiveGrammarException,
|
|
ParseSyntaxException,
|
|
)
|
|
|
|
DEFAULT_EXTRA_CONFIG_PATH = "/opt/trains/config"
|
|
EXTRA_CONFIG_PATH_ENV_KEY = "TRAINS_CONFIG_DIR"
|
|
EXTRA_CONFIG_PATH_SEP = ":"
|
|
|
|
EXTRA_CONFIG_VALUES_ENV_KEY_SEP = "__"
|
|
EXTRA_CONFIG_VALUES_ENV_KEY_PREFIX = f"TRAINS{EXTRA_CONFIG_VALUES_ENV_KEY_SEP}"
|
|
|
|
|
|
class BasicConfig:
|
|
NotSet = object()
|
|
|
|
def __init__(self, folder):
|
|
self.folder = Path(folder)
|
|
if not self.folder.is_dir():
|
|
raise ValueError("Invalid configuration folder")
|
|
|
|
self.prefix = "trains"
|
|
|
|
self._load()
|
|
|
|
def __getitem__(self, key):
|
|
return self._config[key]
|
|
|
|
def get(self, key, default=NotSet):
|
|
value = self._config.get(key, default)
|
|
if value is self.NotSet and not default:
|
|
raise KeyError(
|
|
f"Unable to find value for key '{key}' and default value was not provided."
|
|
)
|
|
return value
|
|
|
|
def logger(self, name):
|
|
if Path(name).is_file():
|
|
name = Path(name).stem
|
|
path = ".".join((self.prefix, Path(name).stem))
|
|
return logging.getLogger(path)
|
|
|
|
def _read_extra_env_config_values(self):
|
|
""" Loads extra configuration from environment-injected values """
|
|
result = ConfigTree()
|
|
prefix = EXTRA_CONFIG_VALUES_ENV_KEY_PREFIX
|
|
|
|
keys = sorted(k for k in os.environ if k.startswith(prefix))
|
|
for key in keys:
|
|
path = key[len(prefix) :].replace(EXTRA_CONFIG_VALUES_ENV_KEY_SEP, ".")
|
|
result = ConfigTree.merge_configs(
|
|
result, ConfigFactory.parse_string(f"{path}: {os.environ[key]}")
|
|
)
|
|
|
|
return result
|
|
|
|
def _read_env_paths(self, key):
|
|
value = getenv(EXTRA_CONFIG_PATH_ENV_KEY, DEFAULT_EXTRA_CONFIG_PATH)
|
|
if value is None:
|
|
return
|
|
paths = [
|
|
Path(expandvars(v)).expanduser() for v in value.split(EXTRA_CONFIG_PATH_SEP)
|
|
]
|
|
invalid = [
|
|
path
|
|
for path in paths
|
|
if not path.is_dir() and str(path) != DEFAULT_EXTRA_CONFIG_PATH
|
|
]
|
|
if invalid:
|
|
print(f"WARNING: Invalid paths in {key} env var: {' '.join(invalid)}")
|
|
return [path for path in paths if path.is_dir()]
|
|
|
|
def _load(self, verbose=True):
|
|
extra_config_paths = self._read_env_paths(EXTRA_CONFIG_PATH_ENV_KEY) or []
|
|
extra_config_values = self._read_extra_env_config_values()
|
|
configs = [
|
|
self._read_recursive(path, verbose=verbose)
|
|
for path in [self.folder] + extra_config_paths
|
|
]
|
|
|
|
self._config = reduce(
|
|
lambda last, config: ConfigTree.merge_configs(
|
|
last, config, copy_trees=True
|
|
),
|
|
configs + [extra_config_values],
|
|
ConfigTree(),
|
|
)
|
|
|
|
def _read_recursive(self, conf_root, verbose=True):
|
|
conf = ConfigTree()
|
|
|
|
if not conf_root:
|
|
return conf
|
|
|
|
if not conf_root.is_dir():
|
|
if verbose:
|
|
if not conf_root.exists():
|
|
print(f"No config in {conf_root}")
|
|
else:
|
|
print(f"Not a directory: {conf_root}")
|
|
return conf
|
|
|
|
if verbose:
|
|
print(f"Loading config from {conf_root}")
|
|
|
|
for file in conf_root.rglob("*.conf"):
|
|
key = ".".join(file.relative_to(conf_root).with_suffix("").parts)
|
|
conf.put(key, self._read_single_file(file, verbose=verbose))
|
|
|
|
return conf
|
|
|
|
@staticmethod
|
|
def _read_single_file(file_path, verbose=True):
|
|
if verbose:
|
|
print(f"Loading config from file {file_path}")
|
|
|
|
try:
|
|
return ConfigFactory.parse_file(file_path)
|
|
except ParseSyntaxException as ex:
|
|
msg = f"Failed parsing {file_path} ({ex.__class__.__name__}): (at char {ex.loc}, line:{ex.lineno}, col:{ex.column})"
|
|
raise ConfigurationError(msg, file_path=file_path) from ex
|
|
except (ParseException, ParseFatalException, RecursiveGrammarException) as ex:
|
|
msg = f"Failed parsing {file_path} ({ex.__class__.__name__}): {ex}"
|
|
raise ConfigurationError(msg) from ex
|
|
except Exception as ex:
|
|
print(f"Failed loading {file_path}: {ex}")
|
|
raise
|
|
|
|
|
|
class ConfigurationError(Exception):
|
|
def __init__(self, msg, file_path=None, *args):
|
|
super(ConfigurationError, self).__init__(msg, *args)
|
|
self.file_path = file_path
|