ClearML/clearml
1
0
Fork 0
mirror of https://github.com/clearml/clearml synced 2025-06-26 18:16:07 +00:00
Code Issues Actions Packages Projects Releases Wiki Activity

Optimize package import time

Browse Source
This commit is contained in:
allegroai 2019-06-14 15:09:51 +03:00
parent 60b2246966
commit 09c7665bfd
2 changed files with 45 additions and 33 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

74
trains/backend_config/config.py
View File

@ -6,9 +6,8 @@ import os
import sys import sys
import warnings import warnings
from fnmatch import fnmatch from fnmatch import fnmatch
from logging import Logger
from os.path import expanduser from os.path import expanduser
from typing import Any, Text from typing import Any
import pyhocon import pyhocon
import six import six
@ -40,6 +39,13 @@ from .log import initialize as initialize_log, logger
from .reloader import ConfigReloader from .reloader import ConfigReloader
from .utils import get_options from .utils import get_options
try:
from typing import Text
except ImportError:
# windows conda-less hack
Text = Any
log = logger(__file__) log = logger(__file__)
@ -150,24 +156,26 @@ class Config(object):
env_config_paths = [expanduser(env_config_path_override)] env_config_paths = [expanduser(env_config_path_override)]
# merge configuration from root and other environment config paths # merge configuration from root and other environment config paths
config = functools.reduce( if self.roots or env_config_paths:
lambda cfg, path: ConfigTree.merge_configs( config = functools.reduce(
cfg, lambda cfg, path: ConfigTree.merge_configs(
self._read_recursive_for_env(path, env, verbose=self._verbose), cfg,
copy_trees=True, self._read_recursive_for_env(path, env, verbose=self._verbose),
), copy_trees=True,
self.roots + env_config_paths, ),
config, self.roots + env_config_paths,
) config,
)
# merge configuration from local configuration paths # merge configuration from local configuration paths
config = functools.reduce( if LOCAL_CONFIG_PATHS:
lambda cfg, path: ConfigTree.merge_configs( config = functools.reduce(
cfg, self._read_recursive(path, verbose=self._verbose), copy_trees=True lambda cfg, path: ConfigTree.merge_configs(
), cfg, self._read_recursive(path, verbose=self._verbose), copy_trees=True
LOCAL_CONFIG_PATHS, ),
config, LOCAL_CONFIG_PATHS,
) config,
)
local_config_files = LOCAL_CONFIG_FILES local_config_files = LOCAL_CONFIG_FILES
local_config_override = os.environ.get(LOCAL_CONFIG_FILE_OVERRIDE_VAR) local_config_override = os.environ.get(LOCAL_CONFIG_FILE_OVERRIDE_VAR)
@ -175,15 +183,16 @@ class Config(object):
local_config_files = [expanduser(local_config_override)] local_config_files = [expanduser(local_config_override)]
# merge configuration from local configuration files # merge configuration from local configuration files
config = functools.reduce( if local_config_files:
lambda cfg, file_path: ConfigTree.merge_configs( config = functools.reduce(
cfg, lambda cfg, file_path: ConfigTree.merge_configs(
self._read_single_file(file_path, verbose=self._verbose), cfg,
copy_trees=True, self._read_single_file(file_path, verbose=self._verbose),
), copy_trees=True,
local_config_files, ),
config, local_config_files,
) config,
)
config["env"] = env config["env"] = env
return config return config
@ -256,10 +265,13 @@ class Config(object):
default_config = self._read_recursive( default_config = self._read_recursive(
root_path / Environment.default, verbose=verbose root_path / Environment.default, verbose=verbose
) )
env_config = self._read_recursive( if (root_path / env) != (root_path / Environment.default):
root_path / env, verbose=verbose env_config = self._read_recursive(
) # None is ok, will return empty config root_path / env, verbose=verbose
config = ConfigTree.merge_configs(default_config, env_config, True) ) # None is ok, will return empty config
config = ConfigTree.merge_configs(default_config, env_config, True)
else:
config = default_config
else: else:
config = ConfigTree() config = ConfigTree()

4
trains/backend_config/defs.py
View File

@ -17,8 +17,8 @@ ENV_CONFIG_PATHS = [
LOCAL_CONFIG_PATHS = [ LOCAL_CONFIG_PATHS = [
'/etc/opt/trains', # used by servers for docker-generated configuration # '/etc/opt/trains', # used by servers for docker-generated configuration
expanduser('~/.trains/config'), # expanduser('~/.trains/config'),
] ]
""" Local config paths, not related to environment """ """ Local config paths, not related to environment """