From 50438bd93120cabd861f37a247366ddbd319bb4b Mon Sep 17 00:00:00 2001 From: allegroai <> Date: Tue, 5 Jan 2021 18:22:39 +0200 Subject: [PATCH] Refactor apierrors infrastructure and auto-generation --- apiserver/apierrors/__init__.py | 137 +----------------- apiserver/apierrors/apierror.py | 7 +- apiserver/apierrors/base.py | 29 +++- apiserver/apierrors/errors.conf | 129 +++++++++++++++++ apiserver/apierrors_generator/__init__.py | 1 + apiserver/apierrors_generator/__main__.py | 4 + .../apierrors_generator/errors_generator.py | 31 ++++ apiserver/apierrors_generator/generator.py | 96 ++++++++++++ .../templates/error.jinja2 | 6 + .../apierrors_generator/templates/init.jinja2 | 14 ++ .../templates/section.jinja2 | 9 ++ 11 files changed, 321 insertions(+), 142 deletions(-) create mode 100644 apiserver/apierrors/errors.conf create mode 100644 apiserver/apierrors_generator/__init__.py create mode 100644 apiserver/apierrors_generator/__main__.py create mode 100644 apiserver/apierrors_generator/errors_generator.py create mode 100644 apiserver/apierrors_generator/generator.py create mode 100644 apiserver/apierrors_generator/templates/error.jinja2 create mode 100644 apiserver/apierrors_generator/templates/init.jinja2 create mode 100644 apiserver/apierrors_generator/templates/section.jinja2 diff --git a/apiserver/apierrors/__init__.py b/apiserver/apierrors/__init__.py index c0b3641..e708c33 100644 --- a/apiserver/apierrors/__init__.py +++ b/apiserver/apierrors/__init__.py @@ -1,137 +1,6 @@ -import pathlib -from . import autogen - from .apierror import APIError +from .base import BaseError +from apiserver.apierrors_generator import ErrorsGenerator -""" Error codes """ -_error_codes = { - (400, 'bad_request'): { - 1: ('not_supported', 'endpoint is not supported'), - 2: ('request_path_has_invalid_version', 'request path has invalid version'), - 5: ('invalid_headers', 'invalid headers'), - 6: ('impersonation_error', 'impersonation error'), - - 10: ('invalid_id', 'invalid object id'), - 11: ('missing_required_fields', 'missing required fields'), - 12: ('validation_error', 'validation error'), - 13: ('fields_not_allowed_for_role', 'fields not allowed for role'), - 14: ('invalid fields', 'fields not defined for object'), - 15: ('fields_conflict', 'conflicting fields'), - 16: ('fields_value_error', 'invalid value for fields'), - 17: ('batch_contains_no_items', 'batch request contains no items'), - 18: ('batch_validation_error', 'batch request validation error'), - 19: ('invalid_lucene_syntax', 'malformed lucene query'), - 20: ('fields_type_error', 'invalid type for fields'), - 21: ('invalid_regex_error', 'malformed regular expression'), - 22: ('invalid_email_address', 'malformed email address'), - 23: ('invalid_domain_name', 'malformed domain name'), - 24: ('not_public_object', 'object is not public'), - - # Tasks - 100: ('task_error', 'general task error'), - 101: ('invalid_task_id', 'invalid task id'), - 102: ('task_validation_error', 'task validation error'), - 110: ('invalid_task_status', 'invalid task status'), - 111: ('task_not_started', 'task not started (invalid task status)'), - 112: ('task_in_progress', 'task in progress (invalid task status)'), - 113: ('task_published', 'task published (invalid task status)'), - 114: ('task_status_unknown', 'task unknown (invalid task status)'), - 120: ('invalid_task_execution_progress', 'invalid task execution progress'), - 121: ('failed_changing_task_status', 'failed changing task status. probably someone changed it before you'), - 122: ('missing_task_fields', 'task is missing expected fields'), - 123: ('task_cannot_be_deleted', 'task cannot be deleted'), - 125: ('task_has_jobs_running', "task has jobs that haven't completed yet"), - 126: ('invalid_task_type', "invalid task type for this operations"), - 127: ('invalid_task_input', 'invalid task output'), - 128: ('invalid_task_output', 'invalid task output'), - 129: ('task_publish_in_progress', 'Task publish in progress'), - 130: ('task_not_found', 'task not found'), - 131: ('events_not_added', 'events not added'), - - # Models - 200: ('model_error', 'general task error'), - 201: ('invalid_model_id', 'invalid model id'), - 202: ('model_not_ready', 'model is not ready'), - 203: ('model_is_ready', 'model is ready'), - 204: ('invalid_model_uri', 'invalid model URI'), - 205: ('model_in_use', 'model is used by tasks'), - 206: ('model_creating_task_exists', 'task that created this model exists'), - - # Users - 300: ('invalid_user', 'invalid user'), - 301: ('invalid_user_id', 'invalid user id'), - 302: ('user_id_exists', 'user id already exists'), - 305: ('invalid_preferences_update', 'Malformed key and/or value'), - - # Projects - 401: ('invalid_project_id', 'invalid project id'), - 402: ('project_has_tasks', 'project has associated tasks'), - 403: ('project_not_found', 'project not found'), - 405: ('project_has_models', 'project has associated models'), - - # Queues - 701: ('invalid_queue_id', 'invalid queue id'), - 702: ('queue_not_empty', 'queue is not empty'), - 703: ('invalid_queue_or_task_not_queued', 'invalid queue id or task not in queue'), - 704: ('removed_during_reposition', 'task was removed by another party during reposition'), - 705: ('failed_adding_during_reposition', 'failed adding task back to queue during reposition'), - 706: ('task_already_queued', 'failed adding task to queue since task is already queued'), - 707: ('no_default_queue', 'no queue is tagged as the default queue for this company'), - 708: ('multiple_default_queues', 'more than one queue is tagged as the default queue for this company'), - - # Database - 800: ('data_validation_error', 'data validation error'), - 801: ('expected_unique_data', 'value combination already exists'), - - # Workers - 1001: ('invalid_worker_id', 'invalid worker id'), - 1002: ('worker_registration_failed', 'worker registration failed'), - 1003: ('worker_registered', 'worker is already registered'), - 1004: ('worker_not_registered', 'worker is not registered'), - 1005: ('worker_stats_not_found', 'worker stats not found'), - - 1104: ('invalid_scroll_id', 'Invalid scroll id'), - }, - - (401, 'unauthorized'): { - 1: ('not_authorized', 'unauthorized (not authorized for endpoint)'), - 2: ('entity_not_allowed', 'unauthorized (entity not allowed)'), - 10: ('bad_auth_type', 'unauthorized (bad authentication header type)'), - 20: ('no_credentials', 'unauthorized (missing credentials)'), - 21: ('bad_credentials', 'unauthorized (malformed credentials)'), - 22: ('invalid_credentials', 'unauthorized (invalid credentials)'), - 30: ('invalid_token', 'invalid token'), - 31: ('blocked_token', 'token is blocked'), - 40: ('invalid_fixed_user', 'fixed user ID was not found') - }, - - (403, 'forbidden'): { - 10: ('routing_error', 'forbidden (routing error)'), - 12: ('blocked_internal_endpoint', 'forbidden (blocked internal endpoint)'), - 20: ('role_not_allowed', 'forbidden (not allowed for role)'), - 21: ('no_write_permission', 'forbidden (modification not allowed)'), - }, - - (500, 'server_error'): { - 0: ('general_error', 'general server error'), - 1: ('internal_error', 'internal server error'), - 2: ('config_error', 'configuration error'), - 3: ('build_info_error', 'build info unavailable or corrupted'), - 4: ('low_disk_space', 'Critical server error! Server reports low or insufficient disk space. Please resolve immediately by allocating additional disk space or freeing up storage space.'), - 10: ('transaction_error', 'a transaction call has returned with an error'), - # Database-related issues - 100: ('data_error', 'general data error'), - 101: ('inconsistent_data', 'inconsistent data encountered in document'), - 102: ('database_unavailable', 'database is temporarily unavailable'), - 110: ('update_failed', 'update failed'), - - # Index-related issues - 201: ('missing_index', 'missing internal index'), - - 9999: ('not_implemented', 'action is not yet implemented'), - } -} - - -autogen.generate(pathlib.Path(__file__).parent, _error_codes) +ErrorsGenerator.generate_python_files() diff --git a/apiserver/apierrors/apierror.py b/apiserver/apierrors/apierror.py index a713610..1986a80 100644 --- a/apiserver/apierrors/apierror.py +++ b/apiserver/apierrors/apierror.py @@ -1,9 +1,10 @@ class APIError(Exception): - def __init__(self, msg, code=500, subcode=0, **_): + def __init__(self, msg, code=500, subcode=0, error_data=None, **_): super(APIError, self).__init__() self._msg = msg self._code = code self._subcode = subcode + self._error_data = error_data or {} @property def msg(self): @@ -17,5 +18,9 @@ class APIError(Exception): def subcode(self): return self._subcode + @property + def error_data(self): + return self._error_data + def __str__(self): return self.msg diff --git a/apiserver/apierrors/base.py b/apiserver/apierrors/base.py index c00f959..6abbfd6 100644 --- a/apiserver/apierrors/base.py +++ b/apiserver/apierrors/base.py @@ -1,9 +1,13 @@ -import six -from boltons.typeutils import classproperty from typing import Tuple +import six +from boltons.iterutils import is_collection, remap +from boltons.typeutils import classproperty + from .apierror import APIError +jsonable_types = (dict, list, tuple, str, int, float, bool, type(None)) + class BaseError(APIError): _default_code = 500 @@ -19,15 +23,26 @@ class BaseError(APIError): f"{k}={self._format_kwarg(v)}" for k, v in kwargs.items() ) message += f": {kwargs_msg}" - params = kwargs.copy() - params.update( - code=self._default_code, subcode=self._default_subcode, msg=message + + super(BaseError, self).__init__( + code=self._default_code, + subcode=self._default_subcode, + msg=message, + error_data=self._to_safe_json_types(kwargs), ) - super(BaseError, self).__init__(**params) + + @staticmethod + def _to_safe_json_types(data): + def visit(_, k, v): + if not isinstance(v, jsonable_types): + v = str(v) + return k, v + + return remap(data, visit=visit) @staticmethod def _format_kwarg(value): - if isinstance(value, (tuple, list)): + if is_collection(value): return f'({", ".join(str(v) for v in value)})' elif isinstance(value, six.string_types): return value diff --git a/apiserver/apierrors/errors.conf b/apiserver/apierrors/errors.conf new file mode 100644 index 0000000..d0a5852 --- /dev/null +++ b/apiserver/apierrors/errors.conf @@ -0,0 +1,129 @@ +400 { + _: "bad_request" + 1: ["not_supported", "endpoint is not supported"] + 2: ["request_path_has_invalid_version", "request path has invalid version"] + 5: ["invalid_headers", "invalid headers"] + 6: ["impersonation_error", "impersonation error"] + + 10: ["invalid_id", "invalid object id"] + 11: ["missing_required_fields", "missing required fields"] + 12: ["validation_error", "validation error"] + 13: ["fields_not_allowed_for_role", "fields not allowed for role"] + 14: ["invalid fields", "fields not defined for object"] + 15: ["fields_conflict", "conflicting fields"] + 16: ["fields_value_error", "invalid value for fields"] + 17: ["batch_contains_no_items", "batch request contains no items"] + 18: ["batch_validation_error", "batch request validation error"] + 19: ["invalid_lucene_syntax", "malformed lucene query"] + 20: ["fields_type_error", "invalid type for fields"] + 21: ["invalid_regex_error", "malformed regular expression"] + 22: ["invalid_email_address", "malformed email address"] + 23: ["invalid_domain_name", "malformed domain name"] + 24: ["not_public_object", "object is not public"] + + # Tasks + 100: ["task_error", "general task error"] + 101: ["invalid_task_id", "invalid task id"] + 102: ["task_validation_error", "task validation error"] + 110: ["invalid_task_status", "invalid task status"] + 111: ["task_not_started", "task not started (invalid task status)"] + 112: ["task_in_progress", "task in progress (invalid task status)"] + 113: ["task_published", "task published (invalid task status)"] + 114: ["task_status_unknown", "task unknown (invalid task status)"] + 120: ["invalid_task_execution_progress", "invalid task execution progress"] + 121: ["failed_changing_task_status", "failed changing task status. probably someone changed it before you"] + 122: ["missing_task_fields", "task is missing expected fields"] + 123: ["task_cannot_be_deleted", "task cannot be deleted"] + 125: ["task_has_jobs_running", "task has jobs that haven't completed yet"] + 126: ["invalid_task_type", "invalid task type for this operations"] + 127: ["invalid_task_input", "invalid task output"] + 128: ["invalid_task_output", "invalid task output"] + 129: ["task_publish_in_progress", "Task publish in progress"] + 130: ["task_not_found", "task not found"] + 131: ["events_not_added", "events not added"] + + # Models + 200: ["model_error", "general task error"] + 201: ["invalid_model_id", "invalid model id"] + 202: ["model_not_ready", "model is not ready"] + 203: ["model_is_ready", "model is ready"] + 204: ["invalid_model_uri", "invalid model URI"] + 205: ["model_in_use", "model is used by tasks"] + 206: ["model_creating_task_exists", "task that created this model exists"] + + # Users + 300: ["invalid_user", "invalid user"] + 301: ["invalid_user_id", "invalid user id"] + 302: ["user_id_exists", "user id already exists"] + 305: ["invalid_preferences_update", "Malformed key and/or value"] + + # Projects + 401: ["invalid_project_id", "invalid project id"] + 402: ["project_has_tasks", "project has associated tasks"] + 403: ["project_not_found", "project not found"] + 405: ["project_has_models", "project has associated models"] + + # Queues + 701: ["invalid_queue_id", "invalid queue id"] + 702: ["queue_not_empty", "queue is not empty"] + 703: ["invalid_queue_or_task_not_queued", "invalid queue id or task not in queue"] + 704: ["removed_during_reposition", "task was removed by another party during reposition"] + 705: ["failed_adding_during_reposition", "failed adding task back to queue during reposition"] + 706: ["task_already_queued", "failed adding task to queue since task is already queued"] + 707: ["no_default_queue", "no queue is tagged as the default queue for this company"] + 708: ["multiple_default_queues", "more than one queue is tagged as the default queue for this company"] + + # Database + 800: ["data_validation_error", "data validation error"] + 801: ["expected_unique_data", "value combination already exists"] + + # Workers + 1001: ["invalid_worker_id", "invalid worker id"] + 1002: ["worker_registration_failed", "worker registration failed"] + 1003: ["worker_registered", "worker is already registered"] + 1004: ["worker_not_registered", "worker is not registered"] + 1005: ["worker_stats_not_found", "worker stats not found"] + + 1104: ["invalid_scroll_id", "Invalid scroll id"] +} + +401 { + _: "unauthorized" + 1: ["not_authorized", "unauthorized (not authorized for endpoint)"] + 2: ["entity_not_allowed", "unauthorized (entity not allowed)"] + 10: ["bad_auth_type", "unauthorized (bad authentication header type)"] + 20: ["no_credentials", "unauthorized (missing credentials)"] + 21: ["bad_credentials", "unauthorized (malformed credentials)"] + 22: ["invalid_credentials", "unauthorized (invalid credentials)"] + 30: ["invalid_token", "invalid token"] + 31: ["blocked_token", "token is blocked"] + 40: ["invalid_fixed_user", "fixed user ID was not found"] +} + +403: { + _: "forbidden" + 10: ["routing_error", "forbidden (routing error)"] + 12: ["blocked_internal_endpoint", "forbidden (blocked internal endpoint)"] + 20: ["role_not_allowed", "forbidden (not allowed for role)"] + 21: ["no_write_permission", "forbidden (modification not allowed)"] +} + +500 { + _: "server_error" + 0: ["general_error", "general server error"] + 1: ["internal_error", "internal server error"] + 2: ["config_error", "configuration error"] + 3: ["build_info_error", "build info unavailable or corrupted"] + 4: ["low_disk_space", "Critical server error! Server reports low or insufficient disk space. Please resolve immediately by allocating additional disk space or freeing up storage space."] + 10: ["transaction_error", "a transaction call has returned with an error"] + # Database-related issues + 100: ["data_error", "general data error"] + 101: ["inconsistent_data", "inconsistent data encountered in document"] + 102: ["database_unavailable", "database is temporarily unavailable"] + 110: ["update_failed", "update failed"] + + # Index-related issues + 201: ["missing_index", "missing internal index"] + + 9999: ["not_implemented", "action is not yet implemented"] +} diff --git a/apiserver/apierrors_generator/__init__.py b/apiserver/apierrors_generator/__init__.py new file mode 100644 index 0000000..4e2c142 --- /dev/null +++ b/apiserver/apierrors_generator/__init__.py @@ -0,0 +1 @@ +from .errors_generator import ErrorsGenerator diff --git a/apiserver/apierrors_generator/__main__.py b/apiserver/apierrors_generator/__main__.py new file mode 100644 index 0000000..3275181 --- /dev/null +++ b/apiserver/apierrors_generator/__main__.py @@ -0,0 +1,4 @@ +from .errors_generator import ErrorsGenerator + +if __name__ == '__main__': + ErrorsGenerator.generate_python_files() diff --git a/apiserver/apierrors_generator/errors_generator.py b/apiserver/apierrors_generator/errors_generator.py new file mode 100644 index 0000000..d114390 --- /dev/null +++ b/apiserver/apierrors_generator/errors_generator.py @@ -0,0 +1,31 @@ +from functools import reduce +from pathlib import Path +from typing import Union + +from pyhocon import ConfigFactory, ConfigTree + +from .generator import Generator + + +class ErrorsGenerator: + _apierrors_path = Path(__file__).parents[1] / "apierrors" + _files = [_apierrors_path / "errors.conf"] + + @classmethod + def _get_codes(cls): + return { + (k, v.pop("_")): v + for k, v in reduce( + ConfigTree.merge_configs, map(ConfigFactory.parse_file, cls._files), + ).items() + } + + @classmethod + def add_errors_file(cls, path: Union[Path, str]): + cls._files.append(path) + + @classmethod + def generate_python_files(cls): + Generator(cls._apierrors_path / "errors", format_pep8=False).make_errors( + cls._get_codes() + ) diff --git a/apiserver/apierrors_generator/generator.py b/apiserver/apierrors_generator/generator.py new file mode 100644 index 0000000..a1a7740 --- /dev/null +++ b/apiserver/apierrors_generator/generator.py @@ -0,0 +1,96 @@ +import re +import json +import jinja2 +import hashlib + +from pathlib import Path + + +env = jinja2.Environment( + loader=jinja2.FileSystemLoader(str(Path(__file__).parent)), + autoescape=jinja2.select_autoescape( + disabled_extensions=("py",), default_for_string=False + ), + trim_blocks=True, + lstrip_blocks=True, +) + + +def env_filter(name=None): + return lambda func: env.filters.setdefault(name or func.__name__, func) + + +@env_filter() +def cls_name(name): + delims = list(map(re.escape, (" ", "_"))) + parts = re.split("|".join(delims), name) + return "".join(x.capitalize() for x in parts) + + +class Generator(object): + _base_class_name = "BaseError" + _base_class_module = "apiserver.apierrors.base" + + def __init__(self, path, format_pep8=True, use_md5=True): + self._use_md5 = use_md5 + self._format_pep8 = format_pep8 + self._path = Path(path) + self._path.mkdir(parents=True, exist_ok=True) + + def _make_init_file(self, path): + (self._path / path / "__init__.py").write_bytes(b"") + + def _do_render(self, file, template, context): + with file.open("w") as f: + result = template.render( + base_class_name=self._base_class_name, + base_class_module=self._base_class_module, + **context + ) + if self._format_pep8: + import autopep8 + + result = autopep8.fix_code( + result, + options={"aggressive": 1, "verbose": 0, "max_line_length": 120}, + ) + f.write(result) + + def _make_section(self, name, code, subcodes): + self._do_render( + file=(self._path / name).with_suffix(".py"), + template=env.get_template("templates/section.jinja2"), + context=dict(code=code, subcodes=list(subcodes.items()),), + ) + + def _make_init(self, sections): + self._do_render( + file=(self._path / "__init__.py"), + template=env.get_template("templates/init.jinja2"), + context=dict(sections=sections,), + ) + + def _key_to_str(self, data): + if isinstance(data, dict): + return {str(k): self._key_to_str(v) for k, v in data.items()} + return data + + def _calc_digest(self, data): + data = json.dumps(self._key_to_str(data), sort_keys=True) + return hashlib.md5(data.encode("utf8")).hexdigest() + + def make_errors(self, errors): + digest = None + digest_file = self._path / "digest.md5" + if self._use_md5: + digest = self._calc_digest(errors) + if digest_file.is_file(): + if digest_file.read_text() == digest: + return + + self._make_init(errors) + for (code, section_name), subcodes in errors.items(): + self._make_section(section_name, int(code), subcodes) + + if self._use_md5: + digest_file.write_text(digest) diff --git a/apiserver/apierrors_generator/templates/error.jinja2 b/apiserver/apierrors_generator/templates/error.jinja2 new file mode 100644 index 0000000..01d6689 --- /dev/null +++ b/apiserver/apierrors_generator/templates/error.jinja2 @@ -0,0 +1,6 @@ +{% macro error_class(name, msg, code, subcode=0) %} +class {{ name }}({{ base_class_name }}): + _default_code = {{ code }} + _default_subcode = {{ subcode }} + _default_msg = "{{ msg|capitalize }}" +{% endmacro -%} \ No newline at end of file diff --git a/apiserver/apierrors_generator/templates/init.jinja2 b/apiserver/apierrors_generator/templates/init.jinja2 new file mode 100644 index 0000000..e16f23f --- /dev/null +++ b/apiserver/apierrors_generator/templates/init.jinja2 @@ -0,0 +1,14 @@ +{% from 'templates/error.jinja2' import error_class with context %} +{% if sections %} +from {{ base_class_module }} import {{ base_class_name }} +{% endif %} + +{% for _, name in sections %} +from . import {{ name }} +{% endfor %} + + +{% for code, name in sections %} +{{ error_class(name|cls_name, name|replace('_', ' '), code) }} + +{% endfor %} diff --git a/apiserver/apierrors_generator/templates/section.jinja2 b/apiserver/apierrors_generator/templates/section.jinja2 new file mode 100644 index 0000000..52478c1 --- /dev/null +++ b/apiserver/apierrors_generator/templates/section.jinja2 @@ -0,0 +1,9 @@ +{% from 'templates/error.jinja2' import error_class with context %} +{% if subcodes %} +from {{ base_class_module }} import {{ base_class_name }} +{% endif %} +{% for subcode, (name, msg) in subcodes %} + + +{{ error_class(name|cls_name, msg, code, subcode|int) -}} +{% endfor %} \ No newline at end of file