ClearML/clearml
1
0
Fork 0
mirror of https://github.com/clearml/clearml synced 2025-06-26 18:16:07 +00:00
Code Issues Actions Packages Projects Releases Wiki Activity

Optimize package import time

Browse Source
This commit is contained in:
allegroai 2019-06-14 15:09:51 +03:00
parent 60b2246966
commit 09c7665bfd
2 changed files with 45 additions and 33 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

16
trains/backend_config/config.py
View File

@ -6,9 +6,8 @@ import os
import sys import sys
import warnings import warnings
from fnmatch import fnmatch from fnmatch import fnmatch
from logging import Logger
from os.path import expanduser from os.path import expanduser
from typing import Any, Text from typing import Any
import pyhocon import pyhocon
import six import six
@ -40,6 +39,13 @@ from .log import initialize as initialize_log, logger
from .reloader import ConfigReloader from .reloader import ConfigReloader
from .utils import get_options from .utils import get_options
try:
from typing import Text
except ImportError:
# windows conda-less hack
Text = Any
log = logger(__file__) log = logger(__file__)
@ -150,6 +156,7 @@ class Config(object):
env_config_paths = [expanduser(env_config_path_override)] env_config_paths = [expanduser(env_config_path_override)]
# merge configuration from root and other environment config paths # merge configuration from root and other environment config paths
if self.roots or env_config_paths:
config = functools.reduce( config = functools.reduce(
lambda cfg, path: ConfigTree.merge_configs( lambda cfg, path: ConfigTree.merge_configs(
cfg, cfg,
@ -161,6 +168,7 @@ class Config(object):
) )
# merge configuration from local configuration paths # merge configuration from local configuration paths
if LOCAL_CONFIG_PATHS:
config = functools.reduce( config = functools.reduce(
lambda cfg, path: ConfigTree.merge_configs( lambda cfg, path: ConfigTree.merge_configs(
cfg, self._read_recursive(path, verbose=self._verbose), copy_trees=True cfg, self._read_recursive(path, verbose=self._verbose), copy_trees=True
@ -175,6 +183,7 @@ class Config(object):
local_config_files = [expanduser(local_config_override)] local_config_files = [expanduser(local_config_override)]
# merge configuration from local configuration files # merge configuration from local configuration files
if local_config_files:
config = functools.reduce( config = functools.reduce(
lambda cfg, file_path: ConfigTree.merge_configs( lambda cfg, file_path: ConfigTree.merge_configs(
cfg, cfg,
@ -256,10 +265,13 @@ class Config(object):
default_config = self._read_recursive( default_config = self._read_recursive(
root_path / Environment.default, verbose=verbose root_path / Environment.default, verbose=verbose
) )
if (root_path / env) != (root_path / Environment.default):
env_config = self._read_recursive( env_config = self._read_recursive(
root_path / env, verbose=verbose root_path / env, verbose=verbose
) # None is ok, will return empty config ) # None is ok, will return empty config
config = ConfigTree.merge_configs(default_config, env_config, True) config = ConfigTree.merge_configs(default_config, env_config, True)
else:
config = default_config
else: else:
config = ConfigTree() config = ConfigTree()

4
trains/backend_config/defs.py
View File

@ -17,8 +17,8 @@ ENV_CONFIG_PATHS = [
LOCAL_CONFIG_PATHS = [ LOCAL_CONFIG_PATHS = [
'/etc/opt/trains', # used by servers for docker-generated configuration # '/etc/opt/trains', # used by servers for docker-generated configuration
expanduser('~/.trains/config'), # expanduser('~/.trains/config'),
] ]
""" Local config paths, not related to environment """ """ Local config paths, not related to environment """