ClearML/clearml-server
1
0
Fork 0
mirror of https://github.com/clearml/clearml-server synced 2025-06-26 23:15:47 +00:00
Code Issues Actions Packages Projects Releases Wiki Activity

Use apiserver namespace

Browse Source
This commit is contained in:
allegroai
2021-01-05 16:28:49 +02:00
parent 22e9c2b7eb
commit f832ea565a
110 changed files with 503 additions and 503 deletions
Download Patch File Download Diff File Expand all files Collapse all files

0
apiserver/__init__.py Normal file
View File

2
apiserver/apierrors/autogen/generator.py
View File

@@ -26,7 +26,7 @@ def cls_name(name):
class Generator(object):
_base_class_name = 'BaseError'
_base_class_module = 'apierrors.base'
_base_class_module = 'apiserver.apierrors.base'
def __init__(self, path, format_pep8=True, use_md5=True):
self._use_md5 = use_md5

4
apiserver/apimodels/__init__.py
View File

@@ -12,8 +12,8 @@ from jsonmodels.validators import Enum as EnumValidator
from luqum.parser import parser, ParseError
from validators import email as email_validator, domain as domain_validator
from apierrors import errors
from utilities.json import loads, dumps
from apiserver.apierrors import errors
from apiserver.utilities.json import loads, dumps
def make_default(field_cls, default_value):

8
apiserver/apimodels/auth.py
View File

@@ -2,10 +2,10 @@ from jsonmodels.fields import IntField, StringField, BoolField, EmbeddedField, D
from jsonmodels.models import Base
from jsonmodels.validators import Max, Enum
from apimodels import ListField, EnumField
from config import config
from database.model.auth import Role
from database.utils import get_options
from apiserver.apimodels import ListField, EnumField
from apiserver.config import config
from apiserver.database.model.auth import Role
from apiserver.database.utils import get_options
class GetTokenRequest(Base):

2
apiserver/apimodels/base.py
View File

@@ -2,7 +2,7 @@ from jsonmodels import models, fields
from jsonmodels.validators import Length
from mongoengine.base import BaseDocument
from apimodels import DictField, ListField
from apiserver.apimodels import DictField, ListField
class MongoengineFieldsDict(DictField):

10
apiserver/apimodels/events.py
View File

@@ -6,11 +6,11 @@ from jsonmodels.fields import StringField, BoolField
from jsonmodels.models import Base
from jsonmodels.validators import Length, Min, Max
from apimodels import ListField, IntField, ActualEnumField
from bll.event.event_metrics import EventType
from bll.event.scalar_key import ScalarKeyEnum
from config import config
from utilities.stringenum import StringEnum
from apiserver.apimodels import ListField, IntField, ActualEnumField
from apiserver.bll.event.event_metrics import EventType
from apiserver.bll.event.scalar_key import ScalarKeyEnum
from apiserver.config import config
from apiserver.utilities.stringenum import StringEnum
class HistogramRequestBase(Base):

6
apiserver/apimodels/models.py
View File

@@ -1,9 +1,9 @@
from jsonmodels import models, fields
from six import string_types
from apimodels import ListField, DictField
from apimodels.base import UpdateResponse
from apimodels.tasks import PublishResponse as TaskPublishResponse
from apiserver.apimodels import ListField, DictField
from apiserver.apimodels.base import UpdateResponse
from apiserver.apimodels.tasks import PublishResponse as TaskPublishResponse
class GetFrameworksRequest(models.Base):

4
apiserver/apimodels/projects.py
View File

@@ -1,7 +1,7 @@
from jsonmodels import models, fields
from apimodels import ListField
from apimodels.organization import TagsRequest
from apiserver.apimodels import ListField
from apiserver.apimodels.organization import TagsRequest
class ProjectReq(models.Base):

2
apiserver/apimodels/queues.py
View File

@@ -2,7 +2,7 @@ from jsonmodels import validators
from jsonmodels.fields import StringField, IntField, BoolField, FloatField
from jsonmodels.models import Base
from apimodels import ListField
from apiserver.apimodels import ListField
class GetDefaultResp(Base):

8
apiserver/apimodels/tasks.py
View File

@@ -5,10 +5,10 @@ from jsonmodels import models
from jsonmodels.fields import StringField, BoolField, IntField, EmbeddedField
from jsonmodels.validators import Enum, Length
from apimodels import DictField, ListField
from apimodels.base import UpdateResponse
from database.model.task.task import TaskType
from database.utils import get_options
from apiserver.apimodels import DictField, ListField
from apiserver.apimodels.base import UpdateResponse
from apiserver.database.model.task.task import TaskType
from apiserver.database.utils import get_options
class ArtifactTypeData(models.Base):

2
apiserver/apimodels/users.py
View File

@@ -1,7 +1,7 @@
from jsonmodels.fields import StringField
from jsonmodels.models import Base
from apimodels import DictField
from apiserver.apimodels import DictField
class CreateRequest(Base):

2
apiserver/apimodels/workers.py
View File

@@ -12,7 +12,7 @@ from jsonmodels.fields import (
)
from jsonmodels.models import Base
from apimodels import make_default, ListField, EnumField, JsonSerializableMixin
from apiserver.apimodels import make_default, ListField, EnumField, JsonSerializableMixin
DEFAULT_TIMEOUT = 10 * 60

18
apiserver/app_sequence.py
View File

@@ -4,20 +4,20 @@ from hashlib import md5
from flask import Flask
from semantic_version import Version
import database
from bll.statistics.stats_reporter import StatisticsReporter
from config import config, info
from elastic.initialize import init_es_data, check_elastic_empty, ElasticConnectionError
from mongo.initialize import (
from apiserver import database
from apiserver.bll.statistics.stats_reporter import StatisticsReporter
from apiserver.config import config, info
from apiserver.elastic.initialize import init_es_data, check_elastic_empty, ElasticConnectionError
from apiserver.mongo.initialize import (
init_mongo_data,
pre_populate_data,
check_mongo_empty,
get_last_server_version,
)
from service_repo import ServiceRepo
from sync import distributed_lock
from updates import check_updates_thread
from utilities.threads_manager import ThreadsManager
from apiserver.service_repo import ServiceRepo
from apiserver.sync import distributed_lock
from apiserver.updates import check_updates_thread
from apiserver.utilities.threads_manager import ThreadsManager
log = config.logger(__file__)

24
apiserver/bll/auth/__init__.py
View File

@@ -1,17 +1,17 @@
from datetime import datetime
import database
from apierrors import errors
from apimodels.auth import GetTokenResponse, CreateUserRequest, Credentials as CredModel
from apimodels.users import CreateRequest as Users_CreateRequest
from bll.user import UserBLL
from config import config
from config.info import get_version, get_build_number
from database.errors import translate_errors_context
from database.model.auth import User, Role, Credentials
from database.model.company import Company
from service_repo import APICall, ServiceRepo
from service_repo.auth import Identity, Token, get_client_id, get_secret_key
from apiserver import database
from apiserver.apierrors import errors
from apiserver.apimodels.auth import GetTokenResponse, CreateUserRequest, Credentials as CredModel
from apiserver.apimodels.users import CreateRequest as Users_CreateRequest
from apiserver.bll.user import UserBLL
from apiserver.config import config
from apiserver.config.info import get_version, get_build_number
from apiserver.database.errors import translate_errors_context
from apiserver.database.model.auth import User, Role, Credentials
from apiserver.database.model.company import Company
from apiserver.service_repo import APICall, ServiceRepo
from apiserver.service_repo.auth import Identity, Token, get_client_id, get_secret_key
log = config.logger("AuthBLL")

18
apiserver/bll/event/debug_images_iterator.py
View File

@@ -13,15 +13,15 @@ from jsonmodels.fields import StringField, ListField, IntField
from jsonmodels.models import Base
from redis import StrictRedis
from apierrors import errors
from apimodels import JsonSerializableMixin
from bll.event.event_metrics import EventMetrics
from bll.redis_cache_manager import RedisCacheManager
from config import config
from database.errors import translate_errors_context
from database.model.task.metrics import MetricEventStats
from database.model.task.task import Task
from timing_context import TimingContext
from apiserver.apierrors import errors
from apiserver.apimodels import JsonSerializableMixin
from apiserver.bll.event.event_metrics import EventMetrics
from apiserver.bll.redis_cache_manager import RedisCacheManager
from apiserver.config import config
from apiserver.database.errors import translate_errors_context
from apiserver.database.model.task.metrics import MetricEventStats
from apiserver.database.model.task.task import Task
from apiserver.timing_context import TimingContext
class VariantScrollState(Base):

28
apiserver/bll/event/event_bll.py
View File

@@ -10,20 +10,20 @@ from elasticsearch import helpers
from mongoengine import Q
from nested_dict import nested_dict
import database.utils as dbutils
import es_factory
from apierrors import errors
from bll.event.debug_images_iterator import DebugImagesIterator
from bll.event.event_metrics import EventMetrics, EventType
from bll.event.log_events_iterator import LogEventsIterator, TaskEventsResult
from bll.task import TaskBLL
from config import config
from database.errors import translate_errors_context
from database.model.task.task import Task, TaskStatus
from redis_manager import redman
from timing_context import TimingContext
from tools import safe_get
from utilities.dicts import flatten_nested_items
from apiserver.database import utils as dbutils
from apiserver import es_factory
from apiserver.apierrors import errors
from apiserver.bll.event.debug_images_iterator import DebugImagesIterator
from apiserver.bll.event.event_metrics import EventMetrics, EventType
from apiserver.bll.event.log_events_iterator import LogEventsIterator, TaskEventsResult
from apiserver.bll.task import TaskBLL
from apiserver.config import config
from apiserver.database.errors import translate_errors_context
from apiserver.database.model.task.task import Task, TaskStatus
from apiserver.redis_manager import redman
from apiserver.timing_context import TimingContext
from apiserver.tools import safe_get
from apiserver.utilities.dicts import flatten_nested_items
# noinspection PyTypeChecker
EVENT_TYPES = set(map(attrgetter("value"), EventType))

14
apiserver/bll/event/event_metrics.py
View File

@@ -10,13 +10,13 @@ from typing import Sequence, Tuple
from elasticsearch import Elasticsearch
from mongoengine import Q
from apierrors import errors
from bll.event.scalar_key import ScalarKey, ScalarKeyEnum
from config import config
from database.errors import translate_errors_context
from database.model.task.task import Task
from timing_context import TimingContext
from tools import safe_get
from apiserver.apierrors import errors
from apiserver.bll.event.scalar_key import ScalarKey, ScalarKeyEnum
from apiserver.config import config
from apiserver.database.errors import translate_errors_context
from apiserver.database.model.task.task import Task
from apiserver.timing_context import TimingContext
from apiserver.tools import safe_get
log = config.logger(__file__)

6
apiserver/bll/event/log_events_iterator.py
View File

@@ -3,9 +3,9 @@ from typing import Optional, Tuple, Sequence
import attr
from elasticsearch import Elasticsearch
from bll.event.event_metrics import EventMetrics
from database.errors import translate_errors_context
from timing_context import TimingContext
from apiserver.bll.event.event_metrics import EventMetrics
from apiserver.database.errors import translate_errors_context
from apiserver.timing_context import TimingContext
@attr.s(auto_attribs=True)

6
apiserver/bll/event/scalar_key.py
View File

@@ -4,9 +4,9 @@ Module for polymorphism over different types of X axes in scalar aggregations
from abc import ABC, abstractmethod
from enum import auto
from utilities.stringenum import StringEnum
from bll.util import extract_properties_to_lists
from config import config
from apiserver.utilities.stringenum import StringEnum
from apiserver.bll.util import extract_properties_to_lists
from apiserver.config import config
log = config.logger(__file__)

4
apiserver/bll/model/__init__.py
View File

@@ -2,8 +2,8 @@ from typing import Optional, Sequence
from mongoengine import Q
from database.model.model import Model
from database.utils import get_company_or_none_constraint
from apiserver.database.model.model import Model
from apiserver.database.utils import get_company_or_none_constraint
class ModelBLL:

12
apiserver/bll/organization/__init__.py
View File

@@ -6,12 +6,12 @@ from typing import Sequence, Union, Type, Dict
from mongoengine import Q
from redis import Redis
from config import config
from database.model.base import GetMixin
from database.model.model import Model
from database.model.task.task import Task
from redis_manager import redman
from utilities import json
from apiserver.config import config
from apiserver.database.model.base import GetMixin
from apiserver.database.model.model import Model
from apiserver.database.model.task.task import Task
from apiserver.redis_manager import redman
from apiserver.utilities import json
log = config.logger(__file__)
_settings_prefix = "services.organization"

8
apiserver/bll/project/project_bll.py
View File

@@ -2,10 +2,10 @@ from typing import Sequence, Optional
from mongoengine import Q
from config import config
from database.model.model import Model
from database.model.task.task import Task
from timing_context import TimingContext
from apiserver.config import config
from apiserver.database.model.model import Model
from apiserver.database.model.task.task import Task
from apiserver.timing_context import TimingContext
log = config.logger(__file__)

2
apiserver/bll/query/builder.py
View File

@@ -1,6 +1,6 @@
from typing import Optional, Sequence, Iterable, Union
from config import config
from apiserver.config import config
log = config.logger(__file__)

16
apiserver/bll/queue/queue_bll.py
View File

@@ -4,14 +4,14 @@ from typing import Callable, Sequence, Optional, Tuple
from elasticsearch import Elasticsearch
import database
import es_factory
from apierrors import errors
from bll.queue.queue_metrics import QueueMetrics
from bll.workers import WorkerBLL
from config import config
from database.errors import translate_errors_context
from database.model.queue import Queue, Entry
from apiserver import database
from apiserver import es_factory
from apiserver.apierrors import errors
from apiserver.bll.queue.queue_metrics import QueueMetrics
from apiserver.bll.workers import WorkerBLL
from apiserver.config import config
from apiserver.database.errors import translate_errors_context
from apiserver.database.model.queue import Queue, Entry
log = config.logger(__file__)

14
apiserver/bll/queue/queue_metrics.py
View File

@@ -5,13 +5,13 @@ from typing import Sequence
import elasticsearch.helpers
from elasticsearch import Elasticsearch
import es_factory
from apierrors.errors import bad_request
from bll.query import Builder as QueryBuilder
from config import config
from database.errors import translate_errors_context
from database.model.queue import Queue, Entry
from timing_context import TimingContext
from apiserver import es_factory
from apiserver.apierrors.errors import bad_request
from apiserver.bll.query import Builder as QueryBuilder
from apiserver.config import config
from apiserver.database.errors import translate_errors_context
from apiserver.database.model.queue import Queue, Entry
from apiserver.timing_context import TimingContext
log = config.logger(__file__)

4
apiserver/bll/redis_cache_manager.py
View File

@@ -3,8 +3,8 @@ from typing import Optional, TypeVar, Generic, Type, Callable
from redis import StrictRedis
import database
from timing_context import TimingContext
from apiserver import database
from apiserver.timing_context import TimingContext
T = TypeVar("T")

2
apiserver/bll/statistics/resource_monitor.py
View File

@@ -6,7 +6,7 @@ from time import sleep
import attr
import psutil
from utilities.threads_manager import ThreadsManager
from apiserver.utilities.threads_manager import ThreadsManager
class ResourceMonitor(Thread):

24
apiserver/bll/statistics/stats_reporter.py
View File

@@ -11,18 +11,18 @@ import requests
from requests.adapters import HTTPAdapter
from requests.packages.urllib3.util.retry import Retry
from bll.query import Builder as QueryBuilder
from bll.util import get_server_uuid
from bll.workers import WorkerStats, WorkerBLL
from config import config
from config.info import get_deployment_type
from database.model import Company, User
from database.model.queue import Queue
from database.model.task.task import Task
from tools import safe_get
from utilities.json import dumps
from utilities.threads_manager import ThreadsManager
from version import __version__ as current_version
from apiserver.bll.query import Builder as QueryBuilder
from apiserver.bll.util import get_server_uuid
from apiserver.bll.workers import WorkerStats, WorkerBLL
from apiserver.config import config
from apiserver.config.info import get_deployment_type
from apiserver.database.model import Company, User
from apiserver.database.model.queue import Queue
from apiserver.database.model.task.task import Task
from apiserver.tools import safe_get
from apiserver.utilities.json import dumps
from apiserver.utilities.threads_manager import ThreadsManager
from apiserver.version import __version__ as current_version
from .resource_monitor import ResourceMonitor
log = config.logger(__file__)

12
apiserver/bll/task/hyperparams.py
View File

@@ -5,17 +5,17 @@ from typing import Sequence, Dict
from boltons import iterutils
from apierrors import errors
from apimodels.tasks import (
from apiserver.apierrors import errors
from apiserver.apimodels.tasks import (
HyperParamKey,
HyperParamItem,
ReplaceHyperparams,
Configuration,
)
from bll.task import TaskBLL
from config import config
from database.model.task.task import ParamsItem, Task, ConfigurationItem, TaskStatus
from utilities.parameter_key_escaper import ParameterKeyEscaper
from apiserver.bll.task import TaskBLL
from apiserver.config import config
from apiserver.database.model.task.task import ParamsItem, Task, ConfigurationItem, TaskStatus
from apiserver.utilities.parameter_key_escaper import ParameterKeyEscaper
log = config.logger(__file__)
task_bll = TaskBLL()

10
apiserver/bll/task/non_responsive_tasks_watchdog.py
View File

@@ -1,11 +1,11 @@
from datetime import timedelta, datetime
from time import sleep
from apierrors import errors
from bll.task import ChangeStatusRequest
from config import config
from database.model.task.task import TaskStatus, Task
from utilities.threads_manager import ThreadsManager
from apiserver.apierrors import errors
from apiserver.bll.task import ChangeStatusRequest
from apiserver.config import config
from apiserver.database.model.task.task import TaskStatus, Task
from apiserver.utilities.threads_manager import ThreadsManager
log = config.logger(__file__)

8
apiserver/bll/task/param_utils.py
View File

@@ -3,10 +3,10 @@ from typing import Sequence, Tuple
import dpath
from apierrors import errors
from database.model.task.task import Task
from tools import safe_get
from utilities.parameter_key_escaper import ParameterKeyEscaper
from apiserver.apierrors import errors
from apiserver.database.model.task.task import Task
from apiserver.tools import safe_get
from apiserver.utilities.parameter_key_escaper import ParameterKeyEscaper
hyperparams_default_section = "Args"

34
apiserver/bll/task/task_bll.py
View File

@@ -11,18 +11,18 @@ import six
from mongoengine import Q
from six import string_types
import database.utils as dbutils
import es_factory
from apierrors import errors
from apimodels.tasks import Artifact as ApiArtifact
from bll.organization import OrgBLL, Tags
from config import config
from database.errors import translate_errors_context
from database.model.model import Model
from database.model.project import Project
from database.model.task.metrics import EventStats, MetricEventStats
from database.model.task.output import Output
from database.model.task.task import (
import apiserver.database.utils as dbutils
from apiserver import es_factory
from apiserver.apierrors import errors
from apiserver.apimodels.tasks import Artifact as ApiArtifact
from apiserver.bll.organization import OrgBLL, Tags
from apiserver.config import config
from apiserver.database.errors import translate_errors_context
from apiserver.database.model.model import Model
from apiserver.database.model.project import Project
from apiserver.database.model.task.metrics import EventStats, MetricEventStats
from apiserver.database.model.task.output import Output
from apiserver.database.model.task.task import (
Task,
TaskStatus,
TaskStatusMessage,
@@ -31,11 +31,11 @@ from database.model.task.task import (
Artifact,
external_task_types,
)
from database.utils import get_company_or_none_constraint, id as create_id
from service_repo import APICall
from timing_context import TimingContext
from utilities.dicts import deep_merge
from utilities.parameter_key_escaper import ParameterKeyEscaper
from apiserver.database.utils import get_company_or_none_constraint, id as create_id
from apiserver.service_repo import APICall
from apiserver.timing_context import TimingContext
from apiserver.utilities.dicts import deep_merge
from apiserver.utilities.parameter_key_escaper import ParameterKeyEscaper
from .param_utils import params_prepare_for_save
from .utils import ChangeStatusRequest, validate_status_change

14
apiserver/bll/task/utils.py
View File

@@ -4,13 +4,13 @@ from typing import TypeVar, Callable, Tuple, Sequence
import attr
import six
from apierrors import errors
from database.errors import translate_errors_context
from database.model.project import Project
from database.model.task.task import Task, TaskStatus, TaskSystemTags
from database.utils import get_options
from timing_context import TimingContext
from utilities.attrs import typed_attrs
from apiserver.apierrors import errors
from apiserver.database.errors import translate_errors_context
from apiserver.database.model.project import Project
from apiserver.database.model.task.task import Task, TaskStatus, TaskSystemTags
from apiserver.database.utils import get_options
from apiserver.timing_context import TimingContext
from apiserver.utilities.attrs import typed_attrs
valid_statuses = get_options(TaskStatus)

8
apiserver/bll/user/__init__.py
View File

@@ -1,7 +1,7 @@
from apierrors import errors
from apimodels.users import CreateRequest
from database.errors import translate_errors_context
from database.model.user import User
from apiserver.apierrors import errors
from apiserver.apimodels.users import CreateRequest
from apiserver.database.errors import translate_errors_context
from apiserver.database.model.user import User
class UserBLL:

4
apiserver/bll/util.py
View File

@@ -2,8 +2,8 @@ import functools
from operator import itemgetter
from typing import Sequence, Optional, Callable, Tuple, Dict, Any, Set
from database.model import AttributedDocument
from database.model.settings import Settings
from apiserver.database.model import AttributedDocument
from apiserver.database.model.settings import Settings
def extract_properties_to_lists(

28
apiserver/bll/workers/__init__.py
View File

@@ -5,10 +5,10 @@ from typing import Sequence, Set, Optional
import attr
import elasticsearch.helpers
import es_factory
from apierrors import APIError
from apierrors.errors import bad_request, server_error
from apimodels.workers import (
from apiserver import es_factory
from apiserver.apierrors import APIError
from apiserver.apierrors.errors import bad_request, server_error
from apiserver.apimodels.workers import (
DEFAULT_TIMEOUT,
IdNameEntry,
WorkerEntry,
@@ -17,16 +17,16 @@ from apimodels.workers import (
QueueEntry,
MachineStats,
)
from config import config
from database.errors import translate_errors_context
from database.model.auth import User
from database.model.company import Company
from database.model.project import Project
from database.model.queue import Queue
from database.model.task.task import Task
from redis_manager import redman
from timing_context import TimingContext
from tools import safe_get
from apiserver.config import config
from apiserver.database.errors import translate_errors_context
from apiserver.database.model.auth import User
from apiserver.database.model.company import Company
from apiserver.database.model.project import Project
from apiserver.database.model.queue import Queue
from apiserver.database.model.task.task import Task
from apiserver.redis_manager import redman
from apiserver.timing_context import TimingContext
from apiserver.tools import safe_get
from .stats import WorkerStats
log = config.logger(__file__)

12
apiserver/bll/workers/stats.py
View File

@@ -3,12 +3,12 @@ from typing import Optional, Sequence
from boltons.iterutils import bucketize
from apierrors.errors import bad_request
from apimodels.workers import AggregationType, GetStatsRequest, StatItem
from bll.query import Builder as QueryBuilder
from config import config
from database.errors import translate_errors_context
from timing_context import TimingContext
from apiserver.apierrors.errors import bad_request
from apiserver.apimodels.workers import AggregationType, GetStatsRequest, StatItem
from apiserver.bll.query import Builder as QueryBuilder
from apiserver.config import config
from apiserver.database.errors import translate_errors_context
from apiserver.timing_context import TimingContext
log = config.logger(__file__)

4
apiserver/config/info.py
View File

@@ -1,9 +1,9 @@
from functools import lru_cache
from os import getenv
from pathlib import Path
from version import __version__
from config import config
from apiserver.config import config
from apiserver.version import __version__
root = Path(__file__).parent.parent

2
apiserver/database/__init__.py
View File

@@ -8,7 +8,7 @@ from jsonmodels.fields import StringField
from mongoengine import register_connection
from mongoengine.connection import get_connection
from config import config
from apiserver.config import config
from .defs import Database
from .utils import get_items

2
apiserver/database/errors.py
View File

@@ -17,7 +17,7 @@ from mongoengine.errors import (
)
from pymongo.errors import PyMongoError, NotMasterError
from apierrors import errors
from apiserver.apierrors import errors
class MakeGetAllQueryError(Exception):

8
apiserver/database/model/__init__.py
View File

@@ -2,10 +2,10 @@ from enum import Enum
from mongoengine import Document, StringField
from apierrors import errors
from database.model.base import DbModelMixin, ABSTRACT_FLAG
from database.model.company import Company
from database.model.user import User
from apiserver.apierrors import errors
from apiserver.database.model.base import DbModelMixin, ABSTRACT_FLAG
from apiserver.database.model.company import Company
from apiserver.database.model.user import User
class AttributedDocument(DbModelMixin, Document):

8
apiserver/database/model/auth.py
View File

@@ -6,10 +6,10 @@ from mongoengine import (
DateTimeField,
)
from database import Database, strict
from database.model import DbModelMixin
from database.model.base import AuthDocument
from database.utils import get_options
from apiserver.database import Database, strict
from apiserver.database.model import DbModelMixin
from apiserver.database.model.base import AuthDocument
from apiserver.database.utils import get_options
class Entities(object):

16
apiserver/database/model/base.py
View File

@@ -8,14 +8,14 @@ from dateutil.parser import parse as parse_datetime
from mongoengine import Q, Document, ListField, StringField
from pymongo.command_cursor import CommandCursor
from apierrors import errors
from apierrors.base import BaseError
from config import config
from database.errors import MakeGetAllQueryError
from database.projection import project_dict, ProjectionHelper
from database.props import PropsMixin
from database.query import RegexQ, RegexWrapper
from database.utils import (
from apiserver.apierrors import errors
from apiserver.apierrors.base import BaseError
from apiserver.config import config
from apiserver.database.errors import MakeGetAllQueryError
from apiserver.database.projection import project_dict, ProjectionHelper
from apiserver.database.props import PropsMixin
from apiserver.database.query import RegexQ, RegexWrapper
from apiserver.database.utils import (
get_company_or_none_constraint,
get_fields_choices,
field_does_not_exist,

6
apiserver/database/model/company.py
View File

@@ -8,9 +8,9 @@ from mongoengine import (
DateTimeField,
)
from database import Database, strict
from database.fields import StrippedStringField
from database.model import DbModelMixin
from apiserver.database import Database, strict
from apiserver.database.fields import StrippedStringField
from apiserver.database.model import DbModelMixin
class ReportStatsOption(EmbeddedDocument):

18
apiserver/database/model/model.py
View File

@@ -1,14 +1,14 @@
from mongoengine import Document, StringField, DateTimeField, BooleanField
from database import Database, strict
from database.fields import StrippedStringField, SafeDictField, SafeSortedListField
from database.model import DbModelMixin
from database.model.base import GetMixin
from database.model.model_labels import ModelLabels
from database.model.company import Company
from database.model.project import Project
from database.model.task.task import Task
from database.model.user import User
from apiserver.database import Database, strict
from apiserver.database.fields import StrippedStringField, SafeDictField, SafeSortedListField
from apiserver.database.model import DbModelMixin
from apiserver.database.model.base import GetMixin
from apiserver.database.model.model_labels import ModelLabels
from apiserver.database.model.company import Company
from apiserver.database.model.project import Project
from apiserver.database.model.task.task import Task
from apiserver.database.model.user import User
class Model(DbModelMixin, Document):

2
apiserver/database/model/model_labels.py
View File

@@ -1,4 +1,4 @@
from database.fields import NoneType, UnionField, SafeMapField
from apiserver.database.fields import NoneType, UnionField, SafeMapField
class ModelLabels(SafeMapField):

8
apiserver/database/model/project.py
View File

@@ -1,9 +1,9 @@
from mongoengine import StringField, DateTimeField, IntField
from database import Database, strict
from database.fields import StrippedStringField, SafeSortedListField
from database.model import AttributedDocument
from database.model.base import GetMixin
from apiserver.database import Database, strict
from apiserver.database.fields import StrippedStringField, SafeSortedListField
from apiserver.database.model import AttributedDocument
from apiserver.database.model.base import GetMixin
class Project(AttributedDocument):

12
apiserver/database/model/queue.py
View File

@@ -6,12 +6,12 @@ from mongoengine import (
EmbeddedDocumentListField,
)
from database import Database, strict
from database.fields import StrippedStringField, SafeSortedListField
from database.model import DbModelMixin
from database.model.base import ProperDictMixin, GetMixin
from database.model.company import Company
from database.model.task.task import Task
from apiserver.database import Database, strict
from apiserver.database.fields import StrippedStringField, SafeSortedListField
from apiserver.database.model import DbModelMixin
from apiserver.database.model.base import ProperDictMixin, GetMixin
from apiserver.database.model.company import Company
from apiserver.database.model.task.task import Task
class Entry(EmbeddedDocument, ProperDictMixin):

4
apiserver/database/model/settings.py
View File

@@ -3,8 +3,8 @@ from typing import Any, Optional, Sequence, Tuple
from mongoengine import Document, StringField, DynamicField, Q
from mongoengine.errors import NotUniqueError
from database import Database, strict
from database.model import DbModelMixin
from apiserver.database import Database, strict
from apiserver.database.model import DbModelMixin
class SettingKeys:

2
apiserver/database/model/task/metrics.py
View File

@@ -6,7 +6,7 @@ from mongoengine import (
EmbeddedDocumentField,
)
from database.fields import SafeMapField
from apiserver.database.fields import SafeMapField
class MetricEvent(EmbeddedDocument):

4
apiserver/database/model/task/output.py
View File

@@ -1,7 +1,7 @@
from mongoengine import EmbeddedDocument, StringField
from database.fields import StrippedStringField
from database.utils import get_options
from apiserver.database.fields import StrippedStringField
from apiserver.database.utils import get_options
class Result(object):

14
apiserver/database/model/task/task.py
View File

@@ -8,8 +8,8 @@ from mongoengine import (
LongField,
)
from database import Database, strict
from database.fields import (
from apiserver.database import Database, strict
from apiserver.database.fields import (
StrippedStringField,
SafeMapField,
SafeDictField,
@@ -17,11 +17,11 @@ from database.fields import (
EmbeddedDocumentSortedListField,
SafeSortedListField,
)
from database.model import AttributedDocument
from database.model.base import ProperDictMixin, GetMixin
from database.model.model_labels import ModelLabels
from database.model.project import Project
from database.utils import get_options
from apiserver.database.model import AttributedDocument
from apiserver.database.model.base import ProperDictMixin, GetMixin
from apiserver.database.model.model_labels import ModelLabels
from apiserver.database.model.project import Project
from apiserver.database.utils import get_options
from .metrics import MetricEvent, MetricEventStats
from .output import Output

8
apiserver/database/model/user.py
View File

@@ -1,9 +1,9 @@
from mongoengine import Document, StringField, DynamicField
from database import Database, strict
from database.model import DbModelMixin
from database.model.base import GetMixin
from database.model.company import Company
from apiserver.database import Database, strict
from apiserver.database.model import DbModelMixin
from apiserver.database.model.base import GetMixin
from apiserver.database.model.company import Company
class User(DbModelMixin, Document):

4
apiserver/database/model/version.py
View File

@@ -1,7 +1,7 @@
from mongoengine import Document, DateTimeField, StringField
from database import Database, strict
from database.model import DbModelMixin
from apiserver.database import Database, strict
from apiserver.database.model import DbModelMixin
class Version(DbModelMixin, Document):

4
apiserver/database/projection.py
View File

@@ -5,8 +5,8 @@ from typing import Sequence, Dict, Callable, Tuple, Any, Type
import dpath.path
from apierrors import errors
from database.props import PropsMixin
from apiserver.apierrors import errors
from apiserver.database.props import PropsMixin
SEP = "."

4
apiserver/database/props.py
View File

@@ -8,12 +8,12 @@ import six
from mongoengine import EmbeddedDocumentField, EmbeddedDocumentListField
from mongoengine.base import get_document, BaseField
from database.fields import (
from apiserver.database.fields import (
LengthRangeEmbeddedDocumentListField,
UniqueEmbeddedDocumentListField,
EmbeddedDocumentSortedListField,
)
from database.utils import get_fields, get_fields_attr
from apiserver.database.utils import get_fields, get_fields_attr
class PropsMixin(object):

6
apiserver/elastic/initialize.py
View File

@@ -5,9 +5,9 @@ from typing import Type, Optional, Sequence, Any, Union
import urllib3.exceptions
from elasticsearch import Elasticsearch, exceptions
import es_factory
from config import config
from elastic.apply_mappings import apply_mappings_to_cluster
from apiserver import es_factory
from apiserver.config import config
from apiserver.elastic.apply_mappings import apply_mappings_to_cluster
log = config.logger(__file__)

2
apiserver/es_factory.py
View File

@@ -4,7 +4,7 @@ from os import getenv
from boltons.iterutils import first
from elasticsearch import Elasticsearch, Transport
from config import config
from apiserver.config import config
log = config.logger(__file__)

8
apiserver/mongo/initialize/__init__.py
View File

@@ -1,10 +1,10 @@
from pathlib import Path
from typing import Sequence, Union
from config import config
from config.info import get_default_company
from database.model.auth import Role
from service_repo.auth.fixed_user import FixedUser
from apiserver.config import config
from apiserver.config.info import get_default_company
from apiserver.database.model.auth import Role
from apiserver.service_repo.auth.fixed_user import FixedUser
from .migration import _apply_migrations, check_mongo_empty, get_last_server_version
from .pre_populate import PrePopulate
from .user import ensure_fixed_user, _ensure_auth_user, _ensure_backend_user

10
apiserver/mongo/initialize/migration.py
View File

@@ -6,9 +6,9 @@ from pathlib import Path
from mongoengine.connection import get_db
from semantic_version import Version
import database.utils
from database import Database
from database.model.version import Version as DatabaseVersion
from apiserver.database import utils
from apiserver.database import Database
from apiserver.database.model.version import Version as DatabaseVersion
migration_dir = Path(__file__).resolve().parent.with_name("migrations")
@@ -16,7 +16,7 @@ migration_dir = Path(__file__).resolve().parent.with_name("migrations")
def check_mongo_empty() -> bool:
return not all(
get_db(alias).collection_names()
for alias in database.utils.get_options(Database)
for alias in utils.get_options(Database)
)
@@ -82,7 +82,7 @@ def _apply_migrations(log: Logger):
)
DatabaseVersion(
id=database.utils.id(),
id=utils.id(),
num=script.stem,
created=datetime.utcnow(),
desc="Applied on server startup",

22
apiserver/mongo/initialize/pre_populate.py
View File

@@ -31,21 +31,21 @@ from boltons.iterutils import chunked_iter
from furl import furl
from mongoengine import Q
from bll.event import EventBLL
from bll.task.param_utils import (
from apiserver.bll.event import EventBLL
from apiserver.bll.task.param_utils import (
split_param_name,
hyperparams_default_section,
hyperparams_legacy_type,
)
from config import config
from config.info import get_default_company
from database.model import EntityVisibility
from database.model.model import Model
from database.model.project import Project
from database.model.task.task import Task, ArtifactModes, TaskStatus
from database.utils import get_options
from tools import safe_get
from utilities import json
from apiserver.config import config
from apiserver.config.info import get_default_company
from apiserver.database.model import EntityVisibility
from apiserver.database.model.model import Model
from apiserver.database.model.project import Project
from apiserver.database.model.task.task import Task, ArtifactModes, TaskStatus
from apiserver.database.utils import get_options
from apiserver.tools import safe_get
from apiserver.utilities import json
from .user import _ensure_backend_user

8
apiserver/mongo/initialize/user.py
View File

@@ -3,10 +3,10 @@ from logging import Logger
import attr
from database.model.auth import Role
from database.model.auth import User as AuthUser, Credentials
from database.model.user import User
from service_repo.auth.fixed_user import FixedUser
from apiserver.database.model.auth import Role
from apiserver.database.model.auth import User as AuthUser, Credentials
from apiserver.database.model.user import User
from apiserver.service_repo.auth.fixed_user import FixedUser
def _ensure_auth_user(user_data: dict, company_id: str, log: Logger, revoke: bool = False):

10
apiserver/mongo/initialize/util.py
View File

@@ -1,11 +1,11 @@
from logging import Logger
from uuid import uuid4
from bll.queue import QueueBLL
from config import config
from database.model.company import Company
from database.model.queue import Queue
from database.model.settings import Settings, SettingKeys
from apiserver.bll.queue import QueueBLL
from apiserver.config import config
from apiserver.database.model.company import Company
from apiserver.database.model.queue import Queue
from apiserver.database.model.settings import Settings, SettingKeys
log = config.logger(__file__)

2
apiserver/mongo/migrations/0.12.1.py
View File

@@ -1,6 +1,6 @@
from pymongo.database import Database, Collection
from database.utils import partition_tags
from apiserver.database.utils import partition_tags
def migrate_backend(db: Database):

2
apiserver/mongo/migrations/0.14.0.py
View File

@@ -2,7 +2,7 @@ import hashlib
from pymongo.database import Database, Collection
from service_repo.auth.fixed_user import FixedUser
from apiserver.service_repo.auth.fixed_user import FixedUser
def _get_ids():

4
apiserver/mongo/migrations/0.16.0.py
View File

@@ -1,11 +1,11 @@
from pymongo.database import Database, Collection
from bll.task.param_utils import (
from apiserver.bll.task.param_utils import (
hyperparams_legacy_type,
hyperparams_default_section,
split_param_name,
)
from tools import safe_get
from apiserver.tools import safe_get
def migrate_backend(db: Database):

4
apiserver/redis_manager.py
View File

@@ -6,8 +6,8 @@ from boltons.iterutils import first
from redis import StrictRedis
from redis.sentinel import Sentinel, SentinelConnectionPool
from apierrors.errors.server_error import ConfigError, GeneralError
from config import config
from apiserver.apierrors.errors.server_error import ConfigError, GeneralError
from apiserver.config import config
log = config.logger(__file__)

4
apiserver/schema.py
View File

@@ -11,8 +11,8 @@ import attr
from boltons.dictutils import subdict
from pyhocon import ConfigFactory
from config import config
from utilities.partial_version import PartialVersion
from apiserver.config import config
from apiserver.utilities.partial_version import PartialVersion
HERE = Path(__file__)

6
apiserver/server.py
View File

@@ -4,9 +4,9 @@ from flask import Flask
from flask_compress import Compress
from flask_cors import CORS
from app_routes import register_routes
from app_sequence import AppSequence
from config import config
from apiserver.app_routes import register_routes
from apiserver.app_sequence import AppSequence
from apiserver.config import config
app = Flask(__name__, static_url_path="/static")
CORS(app, **config.get("apiserver.cors"))

8
apiserver/service_repo/apicall.py
View File

@@ -6,10 +6,10 @@ from typing import Type, Optional
from jsonmodels import models
from six import string_types
import database
from timing_context import TimingContext, TimingStats
from utilities import json
from utilities.partial_version import PartialVersion
from apiserver import database
from apiserver.timing_context import TimingContext, TimingStats
from apiserver.utilities import json
from apiserver.utilities.partial_version import PartialVersion
from .auth import Identity
from .auth import Payload as AuthPayload
from .errors import CallParsingError

14
apiserver/service_repo/auth/auth.py
View File

@@ -4,13 +4,13 @@ from datetime import datetime
import jwt
from mongoengine import Q
from apierrors import errors
from config import config
from database.errors import translate_errors_context
from database.model.auth import User, Entities, Credentials
from database.model.company import Company
from database.utils import get_options
from timing_context import TimingContext
from apiserver.apierrors import errors
from apiserver.config import config
from apiserver.database.errors import translate_errors_context
from apiserver.database.model.auth import User, Entities, Credentials
from apiserver.database.model.company import Company
from apiserver.database.utils import get_options
from apiserver.timing_context import TimingContext
from .fixed_user import FixedUser
from .identity import Identity
from .payload import Payload, Token, Basic, AuthType

4
apiserver/service_repo/auth/fixed_user.py
View File

@@ -4,8 +4,8 @@ from typing import Sequence, Optional
import attr
from config import config
from config.info import get_default_company
from apiserver.config import config
from apiserver.config.info import get_default_company
class FixedUsersError(Exception):

2
apiserver/service_repo/auth/payload/payload.py
View File

@@ -1,4 +1,4 @@
from apierrors import errors
from apiserver.apierrors import errors
from ..identity import Identity
from ..dictable import Dictable

6
apiserver/service_repo/auth/payload/token.py
View File

@@ -2,9 +2,9 @@ import jwt
from datetime import datetime, timedelta
from apierrors import errors
from config import config
from database.model.auth import Role
from apiserver.apierrors import errors
from apiserver.config import config
from apiserver.database.model.auth import Role
from .auth_type import AuthType
from .payload import Payload

4
apiserver/service_repo/endpoint.py
View File

@@ -4,8 +4,8 @@ from boltons.iterutils import remap
from jsonmodels import models
from jsonmodels.errors import FieldNotSupported
from schema import schema
from utilities.partial_version import PartialVersion
from apiserver.schema import schema
from apiserver.utilities.partial_version import PartialVersion
from .apicall import APICall
from .schema_validator import SchemaValidator

4
apiserver/service_repo/schema_validator.py
View File

@@ -6,8 +6,8 @@ import fastjsonschema
import jsonschema
from boltons.iterutils import remap
from apierrors import errors
from config import config
from apiserver.apierrors import errors
from apiserver.config import config
log = config.logger(__file__)

12
apiserver/service_repo/service_repo.py
View File

@@ -6,12 +6,12 @@ from typing import cast, Iterable, List, MutableMapping, Optional, Tuple
import jsonmodels.models
import timing_context
from apierrors import APIError
from apierrors.errors.bad_request import RequestPathHasInvalidVersion
from api_version import __version__ as _api_version_
from config import config
from utilities.partial_version import PartialVersion
from apiserver import timing_context
from apiserver.apierrors import APIError
from apiserver.apierrors.errors.bad_request import RequestPathHasInvalidVersion
from apiserver.api_version import __version__ as _api_version_
from apiserver.config import config
from apiserver.utilities.partial_version import PartialVersion
from .apicall import APICall
from .endpoint import Endpoint
from .errors import MalformedPathError, InvalidVersionError, CallFailedError

14
apiserver/service_repo/validators.py
View File

@@ -1,13 +1,13 @@
import fastjsonschema
import jsonmodels.errors
from apierrors import errors, APIError
from config import config
from database.model import Company
from database.model.auth import Role, User
from service_repo import APICall
from service_repo.apicall import MissingIdentity
from service_repo.endpoint import Endpoint
from apiserver.apierrors import errors, APIError
from apiserver.config import config
from apiserver.database.model import Company
from apiserver.database.model.auth import Role, User
from apiserver.service_repo import APICall
from apiserver.service_repo.apicall import MissingIdentity
from apiserver.service_repo.endpoint import Endpoint
from .auth import get_auth_func, Identity, authorize_impersonation, Payload
from .errors import CallParsingError

20
apiserver/services/auth.py
View File

@@ -1,5 +1,5 @@
from apierrors import errors
from apimodels.auth import (
from apiserver.apierrors import errors
from apiserver.apimodels.auth import (
GetTokenResponse,
GetTokenForUserRequest,
GetTokenRequest,
@@ -14,14 +14,14 @@ from apimodels.auth import (
RevokeCredentialsRequest,
EditUserReq,
)
from apimodels.base import UpdateResponse
from bll.auth import AuthBLL
from config import config, info
from database.errors import translate_errors_context
from database.model.auth import User
from service_repo import APICall, endpoint
from service_repo.auth import Token
from service_repo.auth.fixed_user import FixedUser
from apiserver.apimodels.base import UpdateResponse
from apiserver.bll.auth import AuthBLL
from apiserver.config import config, info
from apiserver.database.errors import translate_errors_context
from apiserver.database.model.auth import User
from apiserver.service_repo import APICall, endpoint
from apiserver.service_repo.auth import Token
from apiserver.service_repo.auth.fixed_user import FixedUser
log = config.logger(__file__)

2
apiserver/services/debug.py
View File

@@ -1,4 +1,4 @@
from service_repo import APICall, endpoint
from apiserver.service_repo import APICall, endpoint
@endpoint("debug.ping")

14
apiserver/services/events.py
View File

@@ -2,8 +2,8 @@ import itertools
from collections import defaultdict
from operator import itemgetter
from apierrors import errors
from apimodels.events import (
from apiserver.apierrors import errors
from apiserver.apimodels.events import (
MultiTaskScalarMetricsIterHistogramRequest,
ScalarMetricsIterHistogramRequest,
DebugImagesRequest,
@@ -14,11 +14,11 @@ from apimodels.events import (
LogEventsRequest,
LogOrderEnum,
)
from bll.event import EventBLL
from bll.event.event_metrics import EventMetrics
from bll.task import TaskBLL
from service_repo import APICall, endpoint
from utilities import json
from apiserver.bll.event import EventBLL
from apiserver.bll.event.event_metrics import EventMetrics
from apiserver.bll.task import TaskBLL
from apiserver.service_repo import APICall, endpoint
from apiserver.utilities import json
task_bll = TaskBLL()
event_bll = EventBLL()

36
apiserver/services/models.py
View File

@@ -3,11 +3,11 @@ from typing import Sequence
from mongoengine import Q, EmbeddedDocument
import database
from apierrors import errors
from apierrors.errors.bad_request import InvalidModelId
from apimodels.base import UpdateResponse, MakePublicRequest
from apimodels.models import (
from apiserver import database
from apiserver.apierrors import errors
from apiserver.apierrors.errors.bad_request import InvalidModelId
from apiserver.apimodels.base import UpdateResponse, MakePublicRequest
from apiserver.apimodels.models import (
CreateModelRequest,
CreateModelResponse,
PublishModelRequest,
@@ -15,23 +15,23 @@ from apimodels.models import (
ModelTaskPublishResponse,
GetFrameworksRequest,
)
from bll.model import ModelBLL
from bll.organization import OrgBLL, Tags
from bll.task import TaskBLL
from config import config
from database.errors import translate_errors_context
from database.model import validate_id
from database.model.model import Model
from database.model.project import Project
from database.model.task.task import Task, TaskStatus
from database.utils import (
from apiserver.bll.model import ModelBLL
from apiserver.bll.organization import OrgBLL, Tags
from apiserver.bll.task import TaskBLL
from apiserver.config import config
from apiserver.database.errors import translate_errors_context
from apiserver.database.model import validate_id
from apiserver.database.model.model import Model
from apiserver.database.model.project import Project
from apiserver.database.model.task.task import Task, TaskStatus
from apiserver.database.utils import (
parse_from_call,
get_company_or_none_constraint,
filter_fields,
)
from service_repo import APICall, endpoint
from services.utils import conform_tag_fields, conform_output_tags
from timing_context import TimingContext
from apiserver.service_repo import APICall, endpoint
from apiserver.services.utils import conform_tag_fields, conform_output_tags
from apiserver.timing_context import TimingContext
log = config.logger(__file__)
org_bll = OrgBLL()

8
apiserver/services/organization.py
View File

@@ -1,9 +1,9 @@
from collections import defaultdict
from apimodels.organization import TagsRequest
from bll.organization import OrgBLL, Tags
from service_repo import endpoint, APICall
from services.utils import get_tags_filter_dictionary, get_tags_response
from apiserver.apimodels.organization import TagsRequest
from apiserver.bll.organization import OrgBLL, Tags
from apiserver.service_repo import endpoint, APICall
from apiserver.services.utils import get_tags_filter_dictionary, get_tags_response
org_bll = OrgBLL()

32
apiserver/services/projects.py
View File

@@ -6,31 +6,31 @@ from operator import itemgetter
import dpath
from mongoengine import Q
import database
from apierrors import errors
from apierrors.errors.bad_request import InvalidProjectId
from apimodels.base import UpdateResponse, MakePublicRequest
from apimodels.projects import (
from apiserver import database
from apiserver.apierrors import errors
from apiserver.apierrors.errors.bad_request import InvalidProjectId
from apiserver.apimodels.base import UpdateResponse, MakePublicRequest
from apiserver.apimodels.projects import (
GetHyperParamReq,
ProjectReq,
ProjectTagsRequest,
)
from bll.organization import OrgBLL, Tags
from bll.task import TaskBLL
from database.errors import translate_errors_context
from database.model import EntityVisibility
from database.model.model import Model
from database.model.project import Project
from database.model.task.task import Task, TaskStatus
from database.utils import parse_from_call, get_options, get_company_or_none_constraint
from service_repo import APICall, endpoint
from services.utils import (
from apiserver.bll.organization import OrgBLL, Tags
from apiserver.bll.task import TaskBLL
from apiserver.database.errors import translate_errors_context
from apiserver.database.model import EntityVisibility
from apiserver.database.model.model import Model
from apiserver.database.model.project import Project
from apiserver.database.model.task.task import Task, TaskStatus
from apiserver.database.utils import parse_from_call, get_options, get_company_or_none_constraint
from apiserver.service_repo import APICall, endpoint
from apiserver.services.utils import (
conform_tag_fields,
conform_output_tags,
get_tags_filter_dictionary,
get_tags_response,
)
from timing_context import TimingContext
from apiserver.timing_context import TimingContext
org_bll = OrgBLL()
task_bll = TaskBLL()

14
apiserver/services/queues.py
View File

@@ -1,5 +1,5 @@
from apimodels.base import UpdateResponse
from apimodels.queues import (
from apiserver.apimodels.base import UpdateResponse
from apiserver.apimodels.queues import (
GetDefaultResp,
CreateRequest,
DeleteRequest,
@@ -12,11 +12,11 @@ from apimodels.queues import (
GetMetricsResponse,
QueueMetrics,
)
from bll.queue import QueueBLL
from bll.util import extract_properties_to_lists
from bll.workers import WorkerBLL
from service_repo import APICall, endpoint
from services.utils import conform_tag_fields, conform_output_tags, conform_tags
from apiserver.bll.queue import QueueBLL
from apiserver.bll.util import extract_properties_to_lists
from apiserver.bll.workers import WorkerBLL
from apiserver.service_repo import APICall, endpoint
from apiserver.services.utils import conform_tag_fields, conform_output_tags, conform_tags
worker_bll = WorkerBLL()
queue_bll = QueueBLL(worker_bll)

20
apiserver/services/server/__init__.py
View File

@@ -2,16 +2,16 @@ from datetime import datetime
from pyhocon.config_tree import NoneValue
from apierrors import errors
from apimodels.server import ReportStatsOptionRequest, ReportStatsOptionResponse
from bll.statistics.stats_reporter import StatisticsReporter
from config import config
from config.info import get_version, get_build_number, get_commit_number
from database.errors import translate_errors_context
from database.model import Company
from database.model.company import ReportStatsOption
from database.model.settings import Settings, SettingKeys
from service_repo import ServiceRepo, APICall, endpoint
from apiserver.apierrors import errors
from apiserver.apimodels.server import ReportStatsOptionRequest, ReportStatsOptionResponse
from apiserver.bll.statistics.stats_reporter import StatisticsReporter
from apiserver.config import config
from apiserver.config.info import get_version, get_build_number, get_commit_number
from apiserver.database.errors import translate_errors_context
from apiserver.database.model import Company
from apiserver.database.model.company import ReportStatsOption
from apiserver.database.model.settings import Settings, SettingKeys
from apiserver.service_repo import ServiceRepo, APICall, endpoint
@endpoint("server.get_stats")

42
apiserver/services/tasks.py
View File

@@ -10,10 +10,10 @@ from mongoengine import EmbeddedDocument, Q
from mongoengine.queryset.transform import COMPARISON_OPERATORS
from pymongo import UpdateOne
from apierrors import errors, APIError
from apierrors.errors.bad_request import InvalidTaskId
from apimodels.base import UpdateResponse, IdResponse, MakePublicRequest
from apimodels.tasks import (
from apiserver.apierrors import errors, APIError
from apiserver.apierrors.errors.bad_request import InvalidTaskId
from apiserver.apimodels.base import UpdateResponse, IdResponse, MakePublicRequest
from apiserver.apimodels.tasks import (
StartedResponse,
ResetResponse,
PublishRequest,
@@ -40,38 +40,38 @@ from apimodels.tasks import (
DeleteConfigurationRequest,
GetConfigurationNamesRequest,
)
from bll.event import EventBLL
from bll.organization import OrgBLL, Tags
from bll.queue import QueueBLL
from bll.task import (
from apiserver.bll.event import EventBLL
from apiserver.bll.organization import OrgBLL, Tags
from apiserver.bll.queue import QueueBLL
from apiserver.bll.task import (
TaskBLL,
ChangeStatusRequest,
update_project_time,
split_by,
)
from bll.task.hyperparams import HyperParams
from bll.task.non_responsive_tasks_watchdog import NonResponsiveTasksWatchdog
from bll.task.param_utils import (
from apiserver.bll.task.hyperparams import HyperParams
from apiserver.bll.task.non_responsive_tasks_watchdog import NonResponsiveTasksWatchdog
from apiserver.bll.task.param_utils import (
params_prepare_for_save,
params_unprepare_from_saved,
escape_paths,
)
from bll.util import SetFieldsResolver
from database.errors import translate_errors_context
from database.model.model import Model
from database.model.task.output import Output
from database.model.task.task import (
from apiserver.bll.util import SetFieldsResolver
from apiserver.database.errors import translate_errors_context
from apiserver.database.model.model import Model
from apiserver.database.model.task.output import Output
from apiserver.database.model.task.task import (
Task,
TaskStatus,
Script,
DEFAULT_LAST_ITERATION,
Execution,
)
from database.utils import get_fields, parse_from_call
from service_repo import APICall, endpoint
from services.utils import conform_tag_fields, conform_output_tags, validate_tags
from timing_context import TimingContext
from utilities.partial_version import PartialVersion
from apiserver.database.utils import get_fields, parse_from_call
from apiserver.service_repo import APICall, endpoint
from apiserver.services.utils import conform_tag_fields, conform_output_tags, validate_tags
from apiserver.timing_context import TimingContext
from apiserver.utilities.partial_version import PartialVersion
task_fields = set(Task.get_fields())
task_script_fields = set(get_fields(Script))

26
apiserver/services/users.py
View File

@@ -5,19 +5,19 @@ import dpath
from boltons.iterutils import remap
from mongoengine import Q
from apierrors import errors
from apimodels.base import UpdateResponse
from apimodels.users import CreateRequest, SetPreferencesRequest
from bll.project import ProjectBLL
from bll.user import UserBLL
from config import config
from database.errors import translate_errors_context
from database.model.auth import Role
from database.model.company import Company
from database.model.user import User
from database.utils import parse_from_call
from service_repo import APICall, endpoint
from utilities.json import loads, dumps
from apiserver.apierrors import errors
from apiserver.apimodels.base import UpdateResponse
from apiserver.apimodels.users import CreateRequest, SetPreferencesRequest
from apiserver.bll.project import ProjectBLL
from apiserver.bll.user import UserBLL
from apiserver.config import config
from apiserver.database.errors import translate_errors_context
from apiserver.database.model.auth import Role
from apiserver.database.model.company import Company
from apiserver.database.model.user import User
from apiserver.database.utils import parse_from_call
from apiserver.service_repo import APICall, endpoint
from apiserver.utilities.json import loads, dumps
log = config.logger(__file__)
project_bll = ProjectBLL()

12
apiserver/services/utils.py
View File

@@ -1,11 +1,11 @@
from typing import Union, Sequence, Tuple
from apierrors import errors
from apimodels.organization import Filter
from database.model.base import GetMixin
from database.utils import partition_tags
from service_repo import APICall
from utilities.partial_version import PartialVersion
from apiserver.apierrors import errors
from apiserver.apimodels.organization import Filter
from apiserver.database.model.base import GetMixin
from apiserver.database.utils import partition_tags
from apiserver.service_repo import APICall
from apiserver.utilities.partial_version import PartialVersion
def get_tags_filter_dictionary(input_: Filter) -> dict:

12
apiserver/services/workers.py
View File

@@ -4,8 +4,8 @@ from typing import Optional, Sequence, Union
from boltons.iterutils import bucketize
from apierrors.errors import bad_request
from apimodels.workers import (
from apiserver.apierrors.errors import bad_request
from apiserver.apimodels.workers import (
WorkerRequest,
StatusReportRequest,
GetAllRequest,
@@ -23,10 +23,10 @@ from apimodels.workers import (
GetActivityReportResponse,
ActivityReportSeries,
)
from bll.util import extract_properties_to_lists
from bll.workers import WorkerBLL
from config import config
from service_repo import APICall, endpoint
from apiserver.bll.util import extract_properties_to_lists
from apiserver.bll.workers import WorkerBLL
from apiserver.config import config
from apiserver.service_repo import APICall, endpoint
log = config.logger(__file__)

2
apiserver/sync.py
View File

@@ -2,7 +2,7 @@ import time
from contextlib import contextmanager
from time import sleep
from redis_manager import redman
from apiserver.redis_manager import redman
_redis = redman.connection("apiserver")

8
apiserver/tests/api_client.py
View File

@@ -4,18 +4,18 @@ import os
import time
from contextlib import contextmanager
from typing import Sequence
from typing import Union, Tuple, Type
import requests
import six
from boltons.typeutils import issubclass
from boltons.iterutils import remap
from boltons.typeutils import issubclass
from pyhocon import ConfigFactory
from requests.adapters import HTTPAdapter
from requests.auth import HTTPBasicAuth
from requests.packages.urllib3.util.retry import Retry
from typing import Union, Tuple, Type
from apierrors.base import BaseError
from pyhocon import ConfigFactory
from apiserver.apierrors.base import BaseError
config = ConfigFactory.parse_file("api_client.conf")

4
apiserver/tests/automated/__init__.py
View File

@@ -5,8 +5,8 @@ from functools import partial
from typing import Iterable
from unittest import TestCase
from tests.api_client import APIClient
from config import config
from apiserver.tests.api_client import APIClient
from apiserver.config import config
log = config.logger(__file__)

2
apiserver/tests/automated/test_entity_ordering.py
View File

@@ -3,7 +3,7 @@ from time import sleep
from typing import Sequence, Mapping
from tests.automated import TestService
from apiserver.tests.automated import TestService
class TestEntityOrdering(TestService):

4
apiserver/tests/automated/test_models.py
View File

@@ -1,5 +1,5 @@
from apierrors.errors.bad_request import InvalidModelId
from tests.automated import TestService
from apiserver.apierrors.errors.bad_request import InvalidModelId
from apiserver.tests.automated import TestService
MODEL_CANNOT_BE_UPDATED_CODES = (400, 203)
TASK_CANNOT_BE_UPDATED_CODES = (400, 110)

2
apiserver/tests/automated/test_project_tags.py
View File

@@ -1,4 +1,4 @@
from tests.automated import TestService
from apiserver.tests.automated import TestService
class TestProjectTags(TestService):

6
apiserver/tests/automated/test_projection.py
View File

@@ -1,6 +1,6 @@
from config import config
from database.model.task.task import TaskStatus
from tests.automated import TestService
from apiserver.config import config
from apiserver.database.model.task.task import TaskStatus
from apiserver.tests.automated import TestService
log = config.logger(__file__)

8
apiserver/tests/automated/test_projects_edit.py
View File

@@ -1,7 +1,7 @@
from apierrors.errors.bad_request import InvalidProjectId
from apierrors.errors.forbidden import NoWritePermission
from config import config
from tests.automated import TestService
from apiserver.apierrors.errors.bad_request import InvalidProjectId
from apiserver.apierrors.errors.forbidden import NoWritePermission
from apiserver.config import config
from apiserver.tests.automated import TestService
log = config.logger(__file__)

4
apiserver/tests/automated/test_queues.py
View File

@@ -4,8 +4,8 @@ from typing import Sequence
from future.backports.datetime import timedelta
from tests.api_client import AttrDict
from tests.automated import TestService, utc_now_tz_aware
from apiserver.tests.api_client import AttrDict
from apiserver.tests.automated import TestService, utc_now_tz_aware
class TestQueues(TestService):

10
apiserver/tests/automated/test_tags.py
View File

@@ -2,11 +2,11 @@ from datetime import datetime
from time import sleep
from typing import Sequence
from apierrors.errors import bad_request
from database.utils import partition_tags
from tests.api_client import APIClient, AttrDict
from tests.automated import TestService
from config import config
from apiserver.apierrors.errors import bad_request
from apiserver.database.utils import partition_tags
from apiserver.tests.api_client import APIClient, AttrDict
from apiserver.tests.automated import TestService
from apiserver.config import config
log = config.logger(__file__)

Some files were not shown because too many files have changed in this diff Show More