Add configuration loader

apiserver/apimodels/auth.py

@@ -3,7 +3,7 @@ from jsonmodels.models import Base
 from jsonmodels.validators import Max, Enum
 
 from apiserver.apimodels import ListField, EnumField
-from apiserver.config import config
+from apiserver.config_repo import config
 from apiserver.database.model.auth import Role
 from apiserver.database.utils import get_options
 

apiserver/apimodels/events.py

@@ -9,7 +9,7 @@ from jsonmodels.validators import Length, Min, Max
 from apiserver.apimodels import ListField, IntField, ActualEnumField
 from apiserver.bll.event.event_metrics import EventType
 from apiserver.bll.event.scalar_key import ScalarKeyEnum
-from apiserver.config import config
+from apiserver.config_repo import config
 from apiserver.utilities.stringenum import StringEnum
 
 

apiserver/app_sequence.py

@@ -6,7 +6,8 @@ from semantic_version import Version
 
 from apiserver.database import database
 from apiserver.bll.statistics.stats_reporter import StatisticsReporter
-from apiserver.config import config, info
+from apiserver.config import info
+from apiserver.config_repo import config
 from apiserver.elastic.initialize import init_es_data, check_elastic_empty, ElasticConnectionError
 from apiserver.mongo.initialize import (
     init_mongo_data,

apiserver/bll/auth/__init__.py

@@ -5,7 +5,7 @@ from apiserver.apierrors import errors
 from apiserver.apimodels.auth import GetTokenResponse, CreateUserRequest, Credentials as CredModel
 from apiserver.apimodels.users import CreateRequest as Users_CreateRequest
 from apiserver.bll.user import UserBLL
-from apiserver.config import config
+from apiserver.config_repo import config
 from apiserver.config.info import get_version, get_build_number
 from apiserver.database.errors import translate_errors_context
 from apiserver.database.model.auth import User, Role, Credentials

apiserver/bll/event/debug_images_iterator.py

@@ -17,7 +17,7 @@ from apiserver.apierrors import errors
 from apiserver.apimodels import JsonSerializableMixin
 from apiserver.bll.event.event_metrics import EventMetrics
 from apiserver.bll.redis_cache_manager import RedisCacheManager
-from apiserver.config import config
+from apiserver.config_repo import config
 from apiserver.database.errors import translate_errors_context
 from apiserver.database.model.task.metrics import MetricEventStats
 from apiserver.database.model.task.task import Task

apiserver/bll/event/event_bll.py

@@ -20,7 +20,7 @@ from apiserver.bll.event.debug_images_iterator import DebugImagesIterator
 from apiserver.bll.event.event_metrics import EventMetrics, EventType
 from apiserver.bll.event.log_events_iterator import LogEventsIterator, TaskEventsResult
 from apiserver.bll.task import TaskBLL
-from apiserver.config import config
+from apiserver.config_repo import config
 from apiserver.database.errors import translate_errors_context
 from apiserver.database.model.task.task import Task, TaskStatus
 from apiserver.redis_manager import redman

apiserver/bll/event/event_metrics.py

@@ -12,7 +12,7 @@ from mongoengine import Q
 
 from apiserver.apierrors import errors
 from apiserver.bll.event.scalar_key import ScalarKey, ScalarKeyEnum
-from apiserver.config import config
+from apiserver.config_repo import config
 from apiserver.database.errors import translate_errors_context
 from apiserver.database.model.task.task import Task
 from apiserver.timing_context import TimingContext

apiserver/bll/event/scalar_key.py

@@ -6,7 +6,7 @@ from enum import auto
 
 from apiserver.utilities.stringenum import StringEnum
 from apiserver.bll.util import extract_properties_to_lists
-from apiserver.config import config
+from apiserver.config_repo import config
 
 log = config.logger(__file__)
 

apiserver/bll/organization/__init__.py

@@ -6,7 +6,7 @@ from typing import Sequence, Union, Type, Dict
 from mongoengine import Q
 from redis import Redis
 
-from apiserver.config import config
+from apiserver.config_repo import config
 from apiserver.database.model.base import GetMixin
 from apiserver.database.model.model import Model
 from apiserver.database.model.task.task import Task

apiserver/bll/project/project_bll.py

@@ -2,7 +2,7 @@ from typing import Sequence, Optional
 
 from mongoengine import Q
 
-from apiserver.config import config
+from apiserver.config_repo import config
 from apiserver.database.model.model import Model
 from apiserver.database.model.task.task import Task
 from apiserver.timing_context import TimingContext

apiserver/bll/query/builder.py

@@ -1,6 +1,6 @@
 from typing import Optional, Sequence, Iterable, Union
 
-from apiserver.config import config
+from apiserver.config_repo import config
 
 log = config.logger(__file__)
 

apiserver/bll/queue/queue_bll.py

@@ -9,7 +9,7 @@ from apiserver.es_factory import es_factory
 from apiserver.apierrors import errors
 from apiserver.bll.queue.queue_metrics import QueueMetrics
 from apiserver.bll.workers import WorkerBLL
-from apiserver.config import config
+from apiserver.config_repo import config
 from apiserver.database.errors import translate_errors_context
 from apiserver.database.model.queue import Queue, Entry
 

apiserver/bll/queue/queue_metrics.py

@@ -8,7 +8,7 @@ from elasticsearch import Elasticsearch
 from apiserver.es_factory import es_factory
 from apiserver.apierrors.errors import bad_request
 from apiserver.bll.query import Builder as QueryBuilder
-from apiserver.config import config
+from apiserver.config_repo import config
 from apiserver.database.errors import translate_errors_context
 from apiserver.database.model.queue import Queue, Entry
 from apiserver.timing_context import TimingContext

apiserver/bll/statistics/stats_reporter.py

@@ -14,7 +14,7 @@ from requests.packages.urllib3.util.retry import Retry
 from apiserver.bll.query import Builder as QueryBuilder
 from apiserver.bll.util import get_server_uuid
 from apiserver.bll.workers import WorkerStats, WorkerBLL
-from apiserver.config import config
+from apiserver.config_repo import config
 from apiserver.config.info import get_deployment_type
 from apiserver.database.model import Company, User
 from apiserver.database.model.queue import Queue

apiserver/bll/task/hyperparams.py

@@ -14,7 +14,7 @@ from apiserver.apimodels.tasks import (
 )
 from apiserver.bll.task import TaskBLL
 from apiserver.bll.task.utils import get_task_for_update
-from apiserver.config import config
+from apiserver.config_repo import config
 from apiserver.database.model.task.task import ParamsItem, Task, ConfigurationItem
 from apiserver.timing_context import TimingContext
 from apiserver.utilities.parameter_key_escaper import ParameterKeyEscaper

apiserver/bll/task/non_responsive_tasks_watchdog.py

@@ -3,7 +3,7 @@ from time import sleep
 
 from apiserver.apierrors import errors
 from apiserver.bll.task import ChangeStatusRequest
-from apiserver.config import config
+from apiserver.config_repo import config
 from apiserver.database.model.task.task import TaskStatus, Task
 from apiserver.utilities.threads_manager import ThreadsManager
 

apiserver/bll/task/task_bll.py

@@ -10,7 +10,7 @@ from six import string_types
 import apiserver.database.utils as dbutils
 from apiserver.apierrors import errors
 from apiserver.bll.organization import OrgBLL, Tags
-from apiserver.config import config
+from apiserver.config_repo import config
 from apiserver.database.errors import translate_errors_context
 from apiserver.database.model.model import Model
 from apiserver.database.model.project import Project

apiserver/bll/workers/__init__.py

@@ -17,7 +17,7 @@ from apiserver.apimodels.workers import (
     QueueEntry,
     MachineStats,
 )
-from apiserver.config import config
+from apiserver.config_repo import config
 from apiserver.database.errors import translate_errors_context
 from apiserver.database.model.auth import User
 from apiserver.database.model.company import Company

apiserver/bll/workers/stats.py

@@ -6,7 +6,7 @@ from boltons.iterutils import bucketize
 from apiserver.apierrors.errors import bad_request
 from apiserver.apimodels.workers import AggregationType, GetStatsRequest, StatItem
 from apiserver.bll.query import Builder as QueryBuilder
-from apiserver.config import config
+from apiserver.config_repo import config
 from apiserver.database.errors import translate_errors_context
 from apiserver.timing_context import TimingContext
 

apiserver/config/__init__.py

@@ -3,6 +3,8 @@ from pathlib import Path
 
 from .basic import BasicConfig
 
-config = BasicConfig(Path(__file__).with_name("default"))
 
+def load_config():
+    config = BasicConfig(Path(__file__).with_name("default"))
     logging.config.dictConfig(config.get("logging"))
+    return config

apiserver/config/info.py

@@ -2,7 +2,7 @@ from functools import lru_cache
 from os import getenv
 from pathlib import Path
 
-from apiserver.config import config
+from apiserver.config_repo import config
 from apiserver.version import __version__
 
 root = Path(__file__).parent.parent

apiserver/config_repo.py

@@ -0,0 +1,3 @@
+from apiserver.config import load_config
+
+config = load_config()

apiserver/database/__init__.py

@@ -8,7 +8,7 @@ from jsonmodels.fields import StringField
 from mongoengine import register_connection
 from mongoengine.connection import get_connection, disconnect
 
-from apiserver.config import config
+from apiserver.config_repo import config
 from .defs import Database
 from .utils import get_items
 

apiserver/database/model/base.py

@@ -10,7 +10,7 @@ from pymongo.command_cursor import CommandCursor
 
 from apiserver.apierrors import errors
 from apiserver.apierrors.base import BaseError
-from apiserver.config import config
+from apiserver.config_repo import config
 from apiserver.database.errors import MakeGetAllQueryError
 from apiserver.database.projection import project_dict, ProjectionHelper
 from apiserver.database.props import PropsMixin

apiserver/elastic/initialize.py

@@ -6,7 +6,7 @@ import urllib3.exceptions
 from elasticsearch import Elasticsearch, exceptions
 
 from apiserver.es_factory import es_factory
-from apiserver.config import config
+from apiserver.config_repo import config
 from apiserver.elastic.apply_mappings import apply_mappings_to_cluster
 
 log = config.logger(__file__)

apiserver/es_factory.py

@@ -4,7 +4,7 @@ from os import getenv
 from boltons.iterutils import first
 from elasticsearch import Elasticsearch, Transport
 
-from apiserver.config import config
+from apiserver.config_repo import config
 
 log = config.logger(__file__)
 

apiserver/mongo/initialize/__init__.py

@@ -1,7 +1,7 @@
 from pathlib import Path
 from typing import Sequence, Union
 
-from apiserver.config import config
+from apiserver.config_repo import config
 from apiserver.config.info import get_default_company
 from apiserver.database.model.auth import Role
 from apiserver.service_repo.auth.fixed_user import FixedUser

apiserver/mongo/initialize/pre_populate.py

@@ -37,7 +37,7 @@ from apiserver.bll.task.param_utils import (
     hyperparams_default_section,
     hyperparams_legacy_type,
 )
-from apiserver.config import config
+from apiserver.config_repo import config
 from apiserver.config.info import get_default_company
 from apiserver.database.model import EntityVisibility
 from apiserver.database.model.model import Model

apiserver/mongo/initialize/util.py

@@ -2,7 +2,7 @@ from logging import Logger
 from uuid import uuid4
 
 from apiserver.bll.queue import QueueBLL
-from apiserver.config import config
+from apiserver.config_repo import config
 from apiserver.database.model.company import Company
 from apiserver.database.model.queue import Queue
 from apiserver.database.model.settings import Settings, SettingKeys

apiserver/redis_manager.py

@@ -7,7 +7,7 @@ from redis import StrictRedis
 from redis.sentinel import Sentinel, SentinelConnectionPool
 
 from apiserver.apierrors.errors.server_error import ConfigError, GeneralError
-from apiserver.config import config
+from apiserver.config_repo import config
 
 log = config.logger(__file__)
 

apiserver/schema.py

@@ -11,7 +11,7 @@ import attr
 from boltons.dictutils import subdict
 from pyhocon import ConfigFactory
 
-from apiserver.config import config
+from apiserver.config_repo import config
 from apiserver.utilities.partial_version import PartialVersion
 
 HERE = Path(__file__)

apiserver/server.py

@@ -7,7 +7,7 @@ from werkzeug.exceptions import BadRequest
 
 from apiserver.apierrors.base import BaseError
 from apiserver.app_sequence import AppSequence
-from apiserver.config import config
+from apiserver.config_repo import config
 from apiserver.service_repo import ServiceRepo, APICall
 from apiserver.service_repo.auth import AuthType
 from apiserver.service_repo.errors import PathParsingError

apiserver/service_repo/auth/auth.py

@@ -5,7 +5,7 @@ import jwt
 from mongoengine import Q
 
 from apiserver.apierrors import errors
-from apiserver.config import config
+from apiserver.config_repo import config
 from apiserver.database.errors import translate_errors_context
 from apiserver.database.model.auth import User, Entities, Credentials
 from apiserver.database.model.company import Company

apiserver/service_repo/auth/fixed_user.py

@@ -4,7 +4,7 @@ from typing import Sequence, Optional
 
 import attr
 
-from apiserver.config import config
+from apiserver.config_repo import config
 from apiserver.config.info import get_default_company
 
 

apiserver/service_repo/auth/payload/token.py

@@ -3,7 +3,7 @@ import jwt
 from datetime import datetime, timedelta
 
 from apiserver.apierrors import errors
-from apiserver.config import config
+from apiserver.config_repo import config
 from apiserver.database.model.auth import Role
 
 from .auth_type import AuthType

apiserver/service_repo/schema_validator.py

@@ -7,7 +7,7 @@ import jsonschema
 from boltons.iterutils import remap
 
 from apiserver.apierrors import errors
-from apiserver.config import config
+from apiserver.config_repo import config
 
 log = config.logger(__file__)
 

apiserver/service_repo/service_repo.py

@@ -10,7 +10,7 @@ from apiserver import timing_context
 from apiserver.apierrors import APIError
 from apiserver.apierrors.errors.bad_request import RequestPathHasInvalidVersion
 from apiserver.api_version import __version__ as _api_version_
-from apiserver.config import config
+from apiserver.config_repo import config
 from apiserver.utilities.partial_version import PartialVersion
 from .apicall import APICall
 from .endpoint import Endpoint

apiserver/service_repo/validators.py

@@ -2,7 +2,7 @@ import fastjsonschema
 import jsonmodels.errors
 
 from apiserver.apierrors import errors, APIError
-from apiserver.config import config
+from apiserver.config_repo import config
 from apiserver.database.model import Company
 from apiserver.database.model.auth import Role, User
 from apiserver.service_repo import APICall

apiserver/services/auth.py

@@ -16,7 +16,8 @@ from apiserver.apimodels.auth import (
 )
 from apiserver.apimodels.base import UpdateResponse
 from apiserver.bll.auth import AuthBLL
-from apiserver.config import config, info
+from apiserver.config import info
+from apiserver.config_repo import config
 from apiserver.database.errors import translate_errors_context
 from apiserver.database.model.auth import User
 from apiserver.service_repo import APICall, endpoint

apiserver/services/models.py

@@ -18,7 +18,7 @@ from apiserver.apimodels.models import (
 from apiserver.bll.model import ModelBLL
 from apiserver.bll.organization import OrgBLL, Tags
 from apiserver.bll.task import TaskBLL
-from apiserver.config import config
+from apiserver.config_repo import config
 from apiserver.database.errors import translate_errors_context
 from apiserver.database.model import validate_id
 from apiserver.database.model.model import Model

apiserver/services/server/__init__.py

@@ -5,7 +5,7 @@ from pyhocon.config_tree import NoneValue
 from apiserver.apierrors import errors
 from apiserver.apimodels.server import ReportStatsOptionRequest, ReportStatsOptionResponse
 from apiserver.bll.statistics.stats_reporter import StatisticsReporter
-from apiserver.config import config
+from apiserver.config_repo import config
 from apiserver.config.info import get_version, get_build_number, get_commit_number
 from apiserver.database.errors import translate_errors_context
 from apiserver.database.model import Company

apiserver/services/users.py

@@ -10,7 +10,7 @@ from apiserver.apimodels.base import UpdateResponse
 from apiserver.apimodels.users import CreateRequest, SetPreferencesRequest
 from apiserver.bll.project import ProjectBLL
 from apiserver.bll.user import UserBLL
-from apiserver.config import config
+from apiserver.config_repo import config
 from apiserver.database.errors import translate_errors_context
 from apiserver.database.model.auth import Role
 from apiserver.database.model.company import Company

apiserver/services/workers.py

@@ -25,7 +25,7 @@ from apiserver.apimodels.workers import (
 )
 from apiserver.bll.util import extract_properties_to_lists
 from apiserver.bll.workers import WorkerBLL
-from apiserver.config import config
+from apiserver.config_repo import config
 from apiserver.service_repo import APICall, endpoint
 
 log = config.logger(__file__)

apiserver/tests/automated/__init__.py

@@ -6,7 +6,7 @@ from typing import Iterable
 from unittest import TestCase
 
 from apiserver.tests.api_client import APIClient
-from apiserver.config import config
+from apiserver.config_repo import config
 
 log = config.logger(__file__)
 

apiserver/tests/automated/test_projection.py

@@ -1,4 +1,4 @@
-from apiserver.config import config
+from apiserver.config_repo import config
 from apiserver.database.model.task.task import TaskStatus
 from apiserver.tests.automated import TestService
 

apiserver/tests/automated/test_projects_edit.py

@@ -1,6 +1,6 @@
 from apiserver.apierrors.errors.bad_request import InvalidProjectId
 from apiserver.apierrors.errors.forbidden import NoWritePermission
-from apiserver.config import config
+from apiserver.config_repo import config
 from apiserver.tests.automated import TestService
 
 

apiserver/tests/automated/test_tags.py

@@ -6,7 +6,7 @@ from apiserver.apierrors.errors import bad_request
 from apiserver.database.utils import partition_tags
 from apiserver.tests.api_client import APIClient, AttrDict
 from apiserver.tests.automated import TestService
-from apiserver.config import config
+from apiserver.config_repo import config
 
 log = config.logger(__file__)
 

apiserver/tests/automated/test_tasks_delete.py

@@ -1,6 +1,6 @@
 from parameterized import parameterized
 
-from apiserver.config import config
+from apiserver.config_repo import config
 from apiserver.tests.automated import TestService
 
 log = config.logger(__file__)

apiserver/tests/automated/test_tasks_diff.py

@@ -1,4 +1,4 @@
-from apiserver.config import config
+from apiserver.config_repo import config
 from apiserver.tests.automated import TestService
 
 log = config.logger(__file__)

apiserver/tests/automated/test_tasks_edit.py

@@ -1,6 +1,6 @@
 from apiserver.apierrors.errors.bad_request import InvalidModelId, ValidationError, InvalidTaskId
 from apiserver.apierrors.errors.forbidden import NoWritePermission
-from apiserver.config import config
+from apiserver.config_repo import config
 from apiserver.tests.automated import TestService
 
 

apiserver/tests/automated/test_users.py

@@ -1,7 +1,7 @@
 from typing import Sequence
 from uuid import uuid4
 
-from apiserver.config import config
+from apiserver.config_repo import config
 from apiserver.tests.automated import TestService
 
 log = config.logger(__file__)

apiserver/tests/automated/test_workers.py

@@ -6,7 +6,7 @@ from typing import Sequence
 
 from apiserver.apierrors.errors import bad_request
 from apiserver.tests.automated import TestService, utc_now_tz_aware
-from apiserver.config import config
+from apiserver.config_repo import config
 
 log = config.logger(__file__)
 

apiserver/updates.py

@@ -7,7 +7,7 @@ import attr
 import requests
 from semantic_version import Version
 
-from apiserver.config import config
+from apiserver.config_repo import config
 from apiserver.config.info import get_version
 from apiserver.database.model.settings import Settings
 from apiserver.utilities.threads_manager import ThreadsManager