Refactor apierrors infrastructure and auto-generation

apiserver/apierrors/__init__.py

@@ -1,137 +1,6 @@
-import pathlib
-from . import autogen
-
 from .apierror import APIError
+from .base import BaseError
 
+from apiserver.apierrors_generator import ErrorsGenerator
 
-""" Error codes """
+ErrorsGenerator.generate_python_files()
-_error_codes = {
-    (400, 'bad_request'): {
-        1: ('not_supported', 'endpoint is not supported'),
-        2: ('request_path_has_invalid_version', 'request path has invalid version'),
-        5: ('invalid_headers', 'invalid headers'),
-        6: ('impersonation_error', 'impersonation error'),
-
-        10: ('invalid_id', 'invalid object id'),
-        11: ('missing_required_fields', 'missing required fields'),
-        12: ('validation_error', 'validation error'),
-        13: ('fields_not_allowed_for_role', 'fields not allowed for role'),
-        14: ('invalid fields', 'fields not defined for object'),
-        15: ('fields_conflict', 'conflicting fields'),
-        16: ('fields_value_error', 'invalid value for fields'),
-        17: ('batch_contains_no_items', 'batch request contains no items'),
-        18: ('batch_validation_error', 'batch request validation error'),
-        19: ('invalid_lucene_syntax', 'malformed lucene query'),
-        20: ('fields_type_error', 'invalid type for fields'),
-        21: ('invalid_regex_error', 'malformed regular expression'),
-        22: ('invalid_email_address', 'malformed email address'),
-        23: ('invalid_domain_name', 'malformed domain name'),
-        24: ('not_public_object', 'object is not public'),
-
-        # Tasks
-        100: ('task_error', 'general task error'),
-        101: ('invalid_task_id', 'invalid task id'),
-        102: ('task_validation_error', 'task validation error'),
-        110: ('invalid_task_status', 'invalid task status'),
-        111: ('task_not_started', 'task not started (invalid task status)'),
-        112: ('task_in_progress', 'task in progress (invalid task status)'),
-        113: ('task_published', 'task published (invalid task status)'),
-        114: ('task_status_unknown', 'task unknown (invalid task status)'),
-        120: ('invalid_task_execution_progress', 'invalid task execution progress'),
-        121: ('failed_changing_task_status', 'failed changing task status. probably someone changed it before you'),
-        122: ('missing_task_fields', 'task is missing expected fields'),
-        123: ('task_cannot_be_deleted', 'task cannot be deleted'),
-        125: ('task_has_jobs_running', "task has jobs that haven't completed yet"),
-        126: ('invalid_task_type', "invalid task type for this operations"),
-        127: ('invalid_task_input', 'invalid task output'),
-        128: ('invalid_task_output', 'invalid task output'),
-        129: ('task_publish_in_progress', 'Task publish in progress'),
-        130: ('task_not_found', 'task not found'),
-        131: ('events_not_added', 'events not added'),
-
-        # Models
-        200: ('model_error', 'general task error'),
-        201: ('invalid_model_id', 'invalid model id'),
-        202: ('model_not_ready', 'model is not ready'),
-        203: ('model_is_ready', 'model is ready'),
-        204: ('invalid_model_uri', 'invalid model URI'),
-        205: ('model_in_use', 'model is used by tasks'),
-        206: ('model_creating_task_exists', 'task that created this model exists'),
-
-        # Users
-        300: ('invalid_user', 'invalid user'),
-        301: ('invalid_user_id', 'invalid user id'),
-        302: ('user_id_exists', 'user id already exists'),
-        305: ('invalid_preferences_update', 'Malformed key and/or value'),
-
-        # Projects
-        401: ('invalid_project_id', 'invalid project id'),
-        402: ('project_has_tasks', 'project has associated tasks'),
-        403: ('project_not_found', 'project not found'),
-        405: ('project_has_models', 'project has associated models'),
-
-        # Queues
-        701: ('invalid_queue_id', 'invalid queue id'),
-        702: ('queue_not_empty', 'queue is not empty'),
-        703: ('invalid_queue_or_task_not_queued', 'invalid queue id or task not in queue'),
-        704: ('removed_during_reposition', 'task was removed by another party during reposition'),
-        705: ('failed_adding_during_reposition', 'failed adding task back to queue during reposition'),
-        706: ('task_already_queued', 'failed adding task to queue since task is already queued'),
-        707: ('no_default_queue', 'no queue is tagged as the default queue for this company'),
-        708: ('multiple_default_queues', 'more than one queue is tagged as the default queue for this company'),
-
-        # Database
-        800: ('data_validation_error', 'data validation error'),
-        801: ('expected_unique_data', 'value combination already exists'),
-
-        # Workers
-        1001: ('invalid_worker_id', 'invalid worker id'),
-        1002: ('worker_registration_failed', 'worker registration failed'),
-        1003: ('worker_registered', 'worker is already registered'),
-        1004: ('worker_not_registered', 'worker is not registered'),
-        1005: ('worker_stats_not_found', 'worker stats not found'),
-
-        1104: ('invalid_scroll_id', 'Invalid scroll id'),
-    },
-
-    (401, 'unauthorized'): {
-        1:  ('not_authorized', 'unauthorized (not authorized for endpoint)'),
-        2:  ('entity_not_allowed', 'unauthorized (entity not allowed)'),
-        10: ('bad_auth_type', 'unauthorized (bad authentication header type)'),
-        20: ('no_credentials', 'unauthorized (missing credentials)'),
-        21: ('bad_credentials', 'unauthorized (malformed credentials)'),
-        22: ('invalid_credentials', 'unauthorized (invalid credentials)'),
-        30: ('invalid_token', 'invalid token'),
-        31: ('blocked_token', 'token is blocked'),
-        40: ('invalid_fixed_user', 'fixed user ID was not found')
-    },
-
-    (403, 'forbidden'): {
-        10: ('routing_error', 'forbidden (routing error)'),
-        12: ('blocked_internal_endpoint', 'forbidden (blocked internal endpoint)'),
-        20: ('role_not_allowed', 'forbidden (not allowed for role)'),
-        21: ('no_write_permission', 'forbidden (modification not allowed)'),
-    },
-
-    (500, 'server_error'): {
-        0:   ('general_error', 'general server error'),
-        1:   ('internal_error', 'internal server error'),
-        2:   ('config_error', 'configuration error'),
-        3:   ('build_info_error', 'build info unavailable or corrupted'),
-        4:   ('low_disk_space', 'Critical server error! Server reports low or insufficient disk space. Please resolve immediately by allocating additional disk space or freeing up storage space.'),
-        10:  ('transaction_error', 'a transaction call has returned with an error'),
-        # Database-related issues
-        100: ('data_error', 'general data error'),
-        101: ('inconsistent_data', 'inconsistent data encountered in document'),
-        102: ('database_unavailable', 'database is temporarily unavailable'),
-        110: ('update_failed', 'update failed'),
-
-        # Index-related issues
-        201: ('missing_index', 'missing internal index'),
-
-        9999: ('not_implemented', 'action is not yet implemented'),
-    }
-}
-
-
-autogen.generate(pathlib.Path(__file__).parent, _error_codes)

apiserver/apierrors/apierror.py

@@ -1,9 +1,10 @@
 class APIError(Exception):
-    def __init__(self, msg, code=500, subcode=0, **_):
+    def __init__(self, msg, code=500, subcode=0, error_data=None, **_):
         super(APIError, self).__init__()
         self._msg = msg
         self._code = code
         self._subcode = subcode
+        self._error_data = error_data or {}
 
     @property
     def msg(self):
@@ -17,5 +18,9 @@ class APIError(Exception):
     def subcode(self):
         return self._subcode
 
+    @property
+    def error_data(self):
+        return self._error_data
+
     def __str__(self):
         return self.msg

apiserver/apierrors/base.py

@@ -1,9 +1,13 @@
-import six
-from boltons.typeutils import classproperty
 from typing import Tuple
 
+import six
+from boltons.iterutils import is_collection, remap
+from boltons.typeutils import classproperty
+
 from .apierror import APIError
 
+jsonable_types = (dict, list, tuple, str, int, float, bool, type(None))
+
 
 class BaseError(APIError):
     _default_code = 500
@@ -19,15 +23,26 @@ class BaseError(APIError):
                 f"{k}={self._format_kwarg(v)}" for k, v in kwargs.items()
             )
             message += f": {kwargs_msg}"
-        params = kwargs.copy()
+
-        params.update(
+        super(BaseError, self).__init__(
-            code=self._default_code, subcode=self._default_subcode, msg=message
+            code=self._default_code,
+            subcode=self._default_subcode,
+            msg=message,
+            error_data=self._to_safe_json_types(kwargs),
         )
-        super(BaseError, self).__init__(**params)
+
+    @staticmethod
+    def _to_safe_json_types(data):
+        def visit(_, k, v):
+            if not isinstance(v, jsonable_types):
+                v = str(v)
+            return k, v
+
+        return remap(data, visit=visit)
 
     @staticmethod
     def _format_kwarg(value):
-        if isinstance(value, (tuple, list)):
+        if is_collection(value):
             return f'({", ".join(str(v) for v in value)})'
         elif isinstance(value, six.string_types):
             return value

apiserver/apierrors/errors.conf

@@ -0,0 +1,129 @@
+400 {
+    _: "bad_request"
+    1: ["not_supported", "endpoint is not supported"]
+    2: ["request_path_has_invalid_version", "request path has invalid version"]
+    5: ["invalid_headers", "invalid headers"]
+    6: ["impersonation_error", "impersonation error"]
+
+    10: ["invalid_id", "invalid object id"]
+    11: ["missing_required_fields", "missing required fields"]
+    12: ["validation_error", "validation error"]
+    13: ["fields_not_allowed_for_role", "fields not allowed for role"]
+    14: ["invalid fields", "fields not defined for object"]
+    15: ["fields_conflict", "conflicting fields"]
+    16: ["fields_value_error", "invalid value for fields"]
+    17: ["batch_contains_no_items", "batch request contains no items"]
+    18: ["batch_validation_error", "batch request validation error"]
+    19: ["invalid_lucene_syntax", "malformed lucene query"]
+    20: ["fields_type_error", "invalid type for fields"]
+    21: ["invalid_regex_error", "malformed regular expression"]
+    22: ["invalid_email_address", "malformed email address"]
+    23: ["invalid_domain_name", "malformed domain name"]
+    24: ["not_public_object", "object is not public"]
+
+    # Tasks
+    100: ["task_error", "general task error"]
+    101: ["invalid_task_id", "invalid task id"]
+    102: ["task_validation_error", "task validation error"]
+    110: ["invalid_task_status", "invalid task status"]
+    111: ["task_not_started", "task not started (invalid task status)"]
+    112: ["task_in_progress", "task in progress (invalid task status)"]
+    113: ["task_published", "task published (invalid task status)"]
+    114: ["task_status_unknown", "task unknown (invalid task status)"]
+    120: ["invalid_task_execution_progress", "invalid task execution progress"]
+    121: ["failed_changing_task_status", "failed changing task status. probably someone changed it before you"]
+    122: ["missing_task_fields", "task is missing expected fields"]
+    123: ["task_cannot_be_deleted", "task cannot be deleted"]
+    125: ["task_has_jobs_running", "task has jobs that haven't completed yet"]
+    126: ["invalid_task_type", "invalid task type for this operations"]
+    127: ["invalid_task_input", "invalid task output"]
+    128: ["invalid_task_output", "invalid task output"]
+    129: ["task_publish_in_progress", "Task publish in progress"]
+    130: ["task_not_found", "task not found"]
+    131: ["events_not_added", "events not added"]
+
+    # Models
+    200: ["model_error", "general task error"]
+    201: ["invalid_model_id", "invalid model id"]
+    202: ["model_not_ready", "model is not ready"]
+    203: ["model_is_ready", "model is ready"]
+    204: ["invalid_model_uri", "invalid model URI"]
+    205: ["model_in_use", "model is used by tasks"]
+    206: ["model_creating_task_exists", "task that created this model exists"]
+
+    # Users
+    300: ["invalid_user", "invalid user"]
+    301: ["invalid_user_id", "invalid user id"]
+    302: ["user_id_exists", "user id already exists"]
+    305: ["invalid_preferences_update", "Malformed key and/or value"]
+
+    # Projects
+    401: ["invalid_project_id", "invalid project id"]
+    402: ["project_has_tasks", "project has associated tasks"]
+    403: ["project_not_found", "project not found"]
+    405: ["project_has_models", "project has associated models"]
+
+    # Queues
+    701: ["invalid_queue_id", "invalid queue id"]
+    702: ["queue_not_empty", "queue is not empty"]
+    703: ["invalid_queue_or_task_not_queued", "invalid queue id or task not in queue"]
+    704: ["removed_during_reposition", "task was removed by another party during reposition"]
+    705: ["failed_adding_during_reposition", "failed adding task back to queue during reposition"]
+    706: ["task_already_queued", "failed adding task to queue since task is already queued"]
+    707: ["no_default_queue", "no queue is tagged as the default queue for this company"]
+    708: ["multiple_default_queues", "more than one queue is tagged as the default queue for this company"]
+
+    # Database
+    800: ["data_validation_error", "data validation error"]
+    801: ["expected_unique_data", "value combination already exists"]
+
+    # Workers
+    1001: ["invalid_worker_id", "invalid worker id"]
+    1002: ["worker_registration_failed", "worker registration failed"]
+    1003: ["worker_registered", "worker is already registered"]
+    1004: ["worker_not_registered", "worker is not registered"]
+    1005: ["worker_stats_not_found", "worker stats not found"]
+
+    1104: ["invalid_scroll_id", "Invalid scroll id"]
+}
+
+401 {
+    _:  "unauthorized"
+    1:  ["not_authorized", "unauthorized (not authorized for endpoint)"]
+    2:  ["entity_not_allowed", "unauthorized (entity not allowed)"]
+    10: ["bad_auth_type", "unauthorized (bad authentication header type)"]
+    20: ["no_credentials", "unauthorized (missing credentials)"]
+    21: ["bad_credentials", "unauthorized (malformed credentials)"]
+    22: ["invalid_credentials", "unauthorized (invalid credentials)"]
+    30: ["invalid_token", "invalid token"]
+    31: ["blocked_token", "token is blocked"]
+    40: ["invalid_fixed_user", "fixed user ID was not found"]
+}
+
+403: {
+    _:  "forbidden"
+    10: ["routing_error", "forbidden (routing error)"]
+    12: ["blocked_internal_endpoint", "forbidden (blocked internal endpoint)"]
+    20: ["role_not_allowed", "forbidden (not allowed for role)"]
+    21: ["no_write_permission", "forbidden (modification not allowed)"]
+}
+
+500 {
+    _:   "server_error"
+    0:   ["general_error", "general server error"]
+    1:   ["internal_error", "internal server error"]
+    2:   ["config_error", "configuration error"]
+    3:   ["build_info_error", "build info unavailable or corrupted"]
+    4:   ["low_disk_space", "Critical server error! Server reports low or insufficient disk space. Please resolve immediately by allocating additional disk space or freeing up storage space."]
+    10:  ["transaction_error", "a transaction call has returned with an error"]
+    # Database-related issues
+    100: ["data_error", "general data error"]
+    101: ["inconsistent_data", "inconsistent data encountered in document"]
+    102: ["database_unavailable", "database is temporarily unavailable"]
+    110: ["update_failed", "update failed"]
+
+    # Index-related issues
+    201: ["missing_index", "missing internal index"]
+
+    9999: ["not_implemented", "action is not yet implemented"]
+}

apiserver/apierrors_generator/__init__.py

@@ -0,0 +1 @@
+from .errors_generator import ErrorsGenerator

apiserver/apierrors_generator/__main__.py

@@ -0,0 +1,4 @@
+from .errors_generator import ErrorsGenerator
+
+if __name__ == '__main__':
+    ErrorsGenerator.generate_python_files()

apiserver/apierrors_generator/errors_generator.py

@@ -0,0 +1,31 @@
+from functools import reduce
+from pathlib import Path
+from typing import Union
+
+from pyhocon import ConfigFactory, ConfigTree
+
+from .generator import Generator
+
+
+class ErrorsGenerator:
+    _apierrors_path = Path(__file__).parents[1] / "apierrors"
+    _files = [_apierrors_path / "errors.conf"]
+
+    @classmethod
+    def _get_codes(cls):
+        return {
+            (k, v.pop("_")): v
+            for k, v in reduce(
+                ConfigTree.merge_configs, map(ConfigFactory.parse_file, cls._files),
+            ).items()
+        }
+
+    @classmethod
+    def add_errors_file(cls, path: Union[Path, str]):
+        cls._files.append(path)
+
+    @classmethod
+    def generate_python_files(cls):
+        Generator(cls._apierrors_path / "errors", format_pep8=False).make_errors(
+            cls._get_codes()
+        )

apiserver/apierrors_generator/generator.py

@@ -0,0 +1,96 @@
+import re
+import json
+import jinja2
+import hashlib
+
+from pathlib import Path
+
+
+env = jinja2.Environment(
+    loader=jinja2.FileSystemLoader(str(Path(__file__).parent)),
+    autoescape=jinja2.select_autoescape(
+        disabled_extensions=("py",), default_for_string=False
+    ),
+    trim_blocks=True,
+    lstrip_blocks=True,
+)
+
+
+def env_filter(name=None):
+    return lambda func: env.filters.setdefault(name or func.__name__, func)
+
+
+@env_filter()
+def cls_name(name):
+    delims = list(map(re.escape, (" ", "_")))
+    parts = re.split("|".join(delims), name)
+    return "".join(x.capitalize() for x in parts)
+
+
+class Generator(object):
+    _base_class_name = "BaseError"
+    _base_class_module = "apiserver.apierrors.base"
+
+    def __init__(self, path, format_pep8=True, use_md5=True):
+        self._use_md5 = use_md5
+        self._format_pep8 = format_pep8
+        self._path = Path(path)
+        self._path.mkdir(parents=True, exist_ok=True)
+
+    def _make_init_file(self, path):
+        (self._path / path / "__init__.py").write_bytes(b"")
+
+    def _do_render(self, file, template, context):
+        with file.open("w") as f:
+            result = template.render(
+                base_class_name=self._base_class_name,
+                base_class_module=self._base_class_module,
+                **context
+            )
+            if self._format_pep8:
+                import autopep8
+
+                result = autopep8.fix_code(
+                    result,
+                    options={"aggressive": 1, "verbose": 0, "max_line_length": 120},
+                )
+            f.write(result)
+
+    def _make_section(self, name, code, subcodes):
+        self._do_render(
+            file=(self._path / name).with_suffix(".py"),
+            template=env.get_template("templates/section.jinja2"),
+            context=dict(code=code, subcodes=list(subcodes.items()),),
+        )
+
+    def _make_init(self, sections):
+        self._do_render(
+            file=(self._path / "__init__.py"),
+            template=env.get_template("templates/init.jinja2"),
+            context=dict(sections=sections,),
+        )
+
+    def _key_to_str(self, data):
+        if isinstance(data, dict):
+            return {str(k): self._key_to_str(v) for k, v in data.items()}
+        return data
+
+    def _calc_digest(self, data):
+        data = json.dumps(self._key_to_str(data), sort_keys=True)
+        return hashlib.md5(data.encode("utf8")).hexdigest()
+
+    def make_errors(self, errors):
+        digest = None
+        digest_file = self._path / "digest.md5"
+        if self._use_md5:
+            digest = self._calc_digest(errors)
+            if digest_file.is_file():
+                if digest_file.read_text() == digest:
+                    return
+
+        self._make_init(errors)
+        for (code, section_name), subcodes in errors.items():
+            self._make_section(section_name, int(code), subcodes)
+
+        if self._use_md5:
+            digest_file.write_text(digest)

apiserver/apierrors_generator/templates/error.jinja2

@@ -0,0 +1,6 @@
+{% macro error_class(name, msg, code, subcode=0) %}
+class {{ name }}({{ base_class_name }}):
+    _default_code = {{ code }}
+    _default_subcode = {{ subcode }}
+    _default_msg = "{{ msg|capitalize }}"
+{% endmacro -%}

apiserver/apierrors_generator/templates/init.jinja2

@@ -0,0 +1,14 @@
+{% from 'templates/error.jinja2' import error_class with context %}
+{% if sections %}
+from {{ base_class_module }} import {{ base_class_name }}
+{% endif %}
+
+{% for _, name in sections %}
+from . import {{ name }}
+{% endfor %}
+
+
+{% for code, name in sections %}
+{{ error_class(name|cls_name, name|replace('_', ' '), code) }}
+
+{% endfor %}

apiserver/apierrors_generator/templates/section.jinja2

@@ -0,0 +1,9 @@
+{% from 'templates/error.jinja2' import error_class with context %}
+{% if subcodes %}
+from {{ base_class_module }} import {{ base_class_name }}
+{% endif %}
+{% for subcode, (name, msg) in subcodes %}
+
+
+{{ error_class(name|cls_name, msg, code, subcode|int) -}}
+{% endfor %}