ClearML/clearml-server
1
0
Fork 0
mirror of https://github.com/clearml/clearml-server synced 2025-03-03 02:33:02 +00:00
Code Issues Actions Packages Projects Releases Wiki Activity

Refactor apierrors infrastructure and auto-generation

Browse Source
This commit is contained in:
allegroai 2021-01-05 18:22:39 +02:00
parent 28daf49c91
commit 50438bd931
11 changed files with 321 additions and 142 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

137
apiserver/apierrors/__init__.py
View File

@ -1,137 +1,6 @@
import pathlib
from . import autogen
from .apierror import APIError
from .base import BaseError
from apiserver.apierrors_generator import ErrorsGenerator
""" Error codes """
_error_codes = {
(400, 'bad_request'): {
1: ('not_supported', 'endpoint is not supported'),
2: ('request_path_has_invalid_version', 'request path has invalid version'),
5: ('invalid_headers', 'invalid headers'),
6: ('impersonation_error', 'impersonation error'),
10: ('invalid_id', 'invalid object id'),
11: ('missing_required_fields', 'missing required fields'),
12: ('validation_error', 'validation error'),
13: ('fields_not_allowed_for_role', 'fields not allowed for role'),
14: ('invalid fields', 'fields not defined for object'),
15: ('fields_conflict', 'conflicting fields'),
16: ('fields_value_error', 'invalid value for fields'),
17: ('batch_contains_no_items', 'batch request contains no items'),
18: ('batch_validation_error', 'batch request validation error'),
19: ('invalid_lucene_syntax', 'malformed lucene query'),
20: ('fields_type_error', 'invalid type for fields'),
21: ('invalid_regex_error', 'malformed regular expression'),
22: ('invalid_email_address', 'malformed email address'),
23: ('invalid_domain_name', 'malformed domain name'),
24: ('not_public_object', 'object is not public'),
# Tasks
100: ('task_error', 'general task error'),
101: ('invalid_task_id', 'invalid task id'),
102: ('task_validation_error', 'task validation error'),
110: ('invalid_task_status', 'invalid task status'),
111: ('task_not_started', 'task not started (invalid task status)'),
112: ('task_in_progress', 'task in progress (invalid task status)'),
113: ('task_published', 'task published (invalid task status)'),
114: ('task_status_unknown', 'task unknown (invalid task status)'),
120: ('invalid_task_execution_progress', 'invalid task execution progress'),
121: ('failed_changing_task_status', 'failed changing task status. probably someone changed it before you'),
122: ('missing_task_fields', 'task is missing expected fields'),
123: ('task_cannot_be_deleted', 'task cannot be deleted'),
125: ('task_has_jobs_running', "task has jobs that haven't completed yet"),
126: ('invalid_task_type', "invalid task type for this operations"),
127: ('invalid_task_input', 'invalid task output'),
128: ('invalid_task_output', 'invalid task output'),
129: ('task_publish_in_progress', 'Task publish in progress'),
130: ('task_not_found', 'task not found'),
131: ('events_not_added', 'events not added'),
# Models
200: ('model_error', 'general task error'),
201: ('invalid_model_id', 'invalid model id'),
202: ('model_not_ready', 'model is not ready'),
203: ('model_is_ready', 'model is ready'),
204: ('invalid_model_uri', 'invalid model URI'),
205: ('model_in_use', 'model is used by tasks'),
206: ('model_creating_task_exists', 'task that created this model exists'),
# Users
300: ('invalid_user', 'invalid user'),
301: ('invalid_user_id', 'invalid user id'),
302: ('user_id_exists', 'user id already exists'),
305: ('invalid_preferences_update', 'Malformed key and/or value'),
# Projects
401: ('invalid_project_id', 'invalid project id'),
402: ('project_has_tasks', 'project has associated tasks'),
403: ('project_not_found', 'project not found'),
405: ('project_has_models', 'project has associated models'),
# Queues
701: ('invalid_queue_id', 'invalid queue id'),
702: ('queue_not_empty', 'queue is not empty'),
703: ('invalid_queue_or_task_not_queued', 'invalid queue id or task not in queue'),
704: ('removed_during_reposition', 'task was removed by another party during reposition'),
705: ('failed_adding_during_reposition', 'failed adding task back to queue during reposition'),
706: ('task_already_queued', 'failed adding task to queue since task is already queued'),
707: ('no_default_queue', 'no queue is tagged as the default queue for this company'),
708: ('multiple_default_queues', 'more than one queue is tagged as the default queue for this company'),
# Database
800: ('data_validation_error', 'data validation error'),
801: ('expected_unique_data', 'value combination already exists'),
# Workers
1001: ('invalid_worker_id', 'invalid worker id'),
1002: ('worker_registration_failed', 'worker registration failed'),
1003: ('worker_registered', 'worker is already registered'),
1004: ('worker_not_registered', 'worker is not registered'),
1005: ('worker_stats_not_found', 'worker stats not found'),
1104: ('invalid_scroll_id', 'Invalid scroll id'),
},
(401, 'unauthorized'): {
1: ('not_authorized', 'unauthorized (not authorized for endpoint)'),
2: ('entity_not_allowed', 'unauthorized (entity not allowed)'),
10: ('bad_auth_type', 'unauthorized (bad authentication header type)'),
20: ('no_credentials', 'unauthorized (missing credentials)'),
21: ('bad_credentials', 'unauthorized (malformed credentials)'),
22: ('invalid_credentials', 'unauthorized (invalid credentials)'),
30: ('invalid_token', 'invalid token'),
31: ('blocked_token', 'token is blocked'),
40: ('invalid_fixed_user', 'fixed user ID was not found')
},
(403, 'forbidden'): {
10: ('routing_error', 'forbidden (routing error)'),
12: ('blocked_internal_endpoint', 'forbidden (blocked internal endpoint)'),
20: ('role_not_allowed', 'forbidden (not allowed for role)'),
21: ('no_write_permission', 'forbidden (modification not allowed)'),
},
(500, 'server_error'): {
0: ('general_error', 'general server error'),
1: ('internal_error', 'internal server error'),
2: ('config_error', 'configuration error'),
3: ('build_info_error', 'build info unavailable or corrupted'),
4: ('low_disk_space', 'Critical server error! Server reports low or insufficient disk space. Please resolve immediately by allocating additional disk space or freeing up storage space.'),
10: ('transaction_error', 'a transaction call has returned with an error'),
# Database-related issues
100: ('data_error', 'general data error'),
101: ('inconsistent_data', 'inconsistent data encountered in document'),
102: ('database_unavailable', 'database is temporarily unavailable'),
110: ('update_failed', 'update failed'),
# Index-related issues
201: ('missing_index', 'missing internal index'),
9999: ('not_implemented', 'action is not yet implemented'),
}
}
autogen.generate(pathlib.Path(__file__).parent, _error_codes)
ErrorsGenerator.generate_python_files()

7
apiserver/apierrors/apierror.py
View File

@ -1,9 +1,10 @@
class APIError(Exception):
def __init__(self, msg, code=500, subcode=0, **_):
def __init__(self, msg, code=500, subcode=0, error_data=None, **_):
super(APIError, self).__init__()
self._msg = msg
self._code = code
self._subcode = subcode
self._error_data = error_data or {}
@property
def msg(self):
@ -17,5 +18,9 @@ class APIError(Exception):
def subcode(self):
return self._subcode
@property
def error_data(self):
return self._error_data
def __str__(self):
return self.msg

29
apiserver/apierrors/base.py
View File

@ -1,9 +1,13 @@
import six
from boltons.typeutils import classproperty
from typing import Tuple
import six
from boltons.iterutils import is_collection, remap
from boltons.typeutils import classproperty
from .apierror import APIError
jsonable_types = (dict, list, tuple, str, int, float, bool, type(None))
class BaseError(APIError):
_default_code = 500
@ -19,15 +23,26 @@ class BaseError(APIError):
f"{k}={self._format_kwarg(v)}" for k, v in kwargs.items()
)
message += f": {kwargs_msg}"
params = kwargs.copy()
params.update(
code=self._default_code, subcode=self._default_subcode, msg=message
super(BaseError, self).__init__(
code=self._default_code,
subcode=self._default_subcode,
msg=message,
error_data=self._to_safe_json_types(kwargs),
)
super(BaseError, self).__init__(**params)
@staticmethod
def _to_safe_json_types(data):
def visit(_, k, v):
if not isinstance(v, jsonable_types):
v = str(v)
return k, v
return remap(data, visit=visit)
@staticmethod
def _format_kwarg(value):
if isinstance(value, (tuple, list)):
if is_collection(value):
return f'({", ".join(str(v) for v in value)})'
elif isinstance(value, six.string_types):
return value

129
apiserver/apierrors/errors.conf Normal file
View File

@ -0,0 +1,129 @@
400 {
_: "bad_request"
1: ["not_supported", "endpoint is not supported"]
2: ["request_path_has_invalid_version", "request path has invalid version"]
5: ["invalid_headers", "invalid headers"]
6: ["impersonation_error", "impersonation error"]
10: ["invalid_id", "invalid object id"]
11: ["missing_required_fields", "missing required fields"]
12: ["validation_error", "validation error"]
13: ["fields_not_allowed_for_role", "fields not allowed for role"]
14: ["invalid fields", "fields not defined for object"]
15: ["fields_conflict", "conflicting fields"]
16: ["fields_value_error", "invalid value for fields"]
17: ["batch_contains_no_items", "batch request contains no items"]
18: ["batch_validation_error", "batch request validation error"]
19: ["invalid_lucene_syntax", "malformed lucene query"]
20: ["fields_type_error", "invalid type for fields"]
21: ["invalid_regex_error", "malformed regular expression"]
22: ["invalid_email_address", "malformed email address"]
23: ["invalid_domain_name", "malformed domain name"]
24: ["not_public_object", "object is not public"]
# Tasks
100: ["task_error", "general task error"]
101: ["invalid_task_id", "invalid task id"]
102: ["task_validation_error", "task validation error"]
110: ["invalid_task_status", "invalid task status"]
111: ["task_not_started", "task not started (invalid task status)"]
112: ["task_in_progress", "task in progress (invalid task status)"]
113: ["task_published", "task published (invalid task status)"]
114: ["task_status_unknown", "task unknown (invalid task status)"]
120: ["invalid_task_execution_progress", "invalid task execution progress"]
121: ["failed_changing_task_status", "failed changing task status. probably someone changed it before you"]
122: ["missing_task_fields", "task is missing expected fields"]
123: ["task_cannot_be_deleted", "task cannot be deleted"]
125: ["task_has_jobs_running", "task has jobs that haven't completed yet"]
126: ["invalid_task_type", "invalid task type for this operations"]
127: ["invalid_task_input", "invalid task output"]
128: ["invalid_task_output", "invalid task output"]
129: ["task_publish_in_progress", "Task publish in progress"]
130: ["task_not_found", "task not found"]
131: ["events_not_added", "events not added"]
# Models
200: ["model_error", "general task error"]
201: ["invalid_model_id", "invalid model id"]
202: ["model_not_ready", "model is not ready"]
203: ["model_is_ready", "model is ready"]
204: ["invalid_model_uri", "invalid model URI"]
205: ["model_in_use", "model is used by tasks"]
206: ["model_creating_task_exists", "task that created this model exists"]
# Users
300: ["invalid_user", "invalid user"]
301: ["invalid_user_id", "invalid user id"]
302: ["user_id_exists", "user id already exists"]
305: ["invalid_preferences_update", "Malformed key and/or value"]
# Projects
401: ["invalid_project_id", "invalid project id"]
402: ["project_has_tasks", "project has associated tasks"]
403: ["project_not_found", "project not found"]
405: ["project_has_models", "project has associated models"]
# Queues
701: ["invalid_queue_id", "invalid queue id"]
702: ["queue_not_empty", "queue is not empty"]
703: ["invalid_queue_or_task_not_queued", "invalid queue id or task not in queue"]
704: ["removed_during_reposition", "task was removed by another party during reposition"]
705: ["failed_adding_during_reposition", "failed adding task back to queue during reposition"]
706: ["task_already_queued", "failed adding task to queue since task is already queued"]
707: ["no_default_queue", "no queue is tagged as the default queue for this company"]
708: ["multiple_default_queues", "more than one queue is tagged as the default queue for this company"]
# Database
800: ["data_validation_error", "data validation error"]
801: ["expected_unique_data", "value combination already exists"]
# Workers
1001: ["invalid_worker_id", "invalid worker id"]
1002: ["worker_registration_failed", "worker registration failed"]
1003: ["worker_registered", "worker is already registered"]
1004: ["worker_not_registered", "worker is not registered"]
1005: ["worker_stats_not_found", "worker stats not found"]
1104: ["invalid_scroll_id", "Invalid scroll id"]
}
401 {
_: "unauthorized"
1: ["not_authorized", "unauthorized (not authorized for endpoint)"]
2: ["entity_not_allowed", "unauthorized (entity not allowed)"]
10: ["bad_auth_type", "unauthorized (bad authentication header type)"]
20: ["no_credentials", "unauthorized (missing credentials)"]
21: ["bad_credentials", "unauthorized (malformed credentials)"]
22: ["invalid_credentials", "unauthorized (invalid credentials)"]
30: ["invalid_token", "invalid token"]
31: ["blocked_token", "token is blocked"]
40: ["invalid_fixed_user", "fixed user ID was not found"]
}
403: {
_: "forbidden"
10: ["routing_error", "forbidden (routing error)"]
12: ["blocked_internal_endpoint", "forbidden (blocked internal endpoint)"]
20: ["role_not_allowed", "forbidden (not allowed for role)"]
21: ["no_write_permission", "forbidden (modification not allowed)"]
}
500 {
_: "server_error"
0: ["general_error", "general server error"]
1: ["internal_error", "internal server error"]
2: ["config_error", "configuration error"]
3: ["build_info_error", "build info unavailable or corrupted"]
4: ["low_disk_space", "Critical server error! Server reports low or insufficient disk space. Please resolve immediately by allocating additional disk space or freeing up storage space."]
10: ["transaction_error", "a transaction call has returned with an error"]
# Database-related issues
100: ["data_error", "general data error"]
101: ["inconsistent_data", "inconsistent data encountered in document"]
102: ["database_unavailable", "database is temporarily unavailable"]
110: ["update_failed", "update failed"]
# Index-related issues
201: ["missing_index", "missing internal index"]
9999: ["not_implemented", "action is not yet implemented"]
}

1
apiserver/apierrors_generator/__init__.py Normal file
View File

@ -0,0 +1 @@
from .errors_generator import ErrorsGenerator

4
apiserver/apierrors_generator/__main__.py Normal file
View File

@ -0,0 +1,4 @@
from .errors_generator import ErrorsGenerator
if __name__ == '__main__':
ErrorsGenerator.generate_python_files()

31
apiserver/apierrors_generator/errors_generator.py Normal file
View File

@ -0,0 +1,31 @@
from functools import reduce
from pathlib import Path
from typing import Union
from pyhocon import ConfigFactory, ConfigTree
from .generator import Generator
class ErrorsGenerator:
_apierrors_path = Path(__file__).parents[1] / "apierrors"
_files = [_apierrors_path / "errors.conf"]
@classmethod
def _get_codes(cls):
return {
(k, v.pop("_")): v
for k, v in reduce(
ConfigTree.merge_configs, map(ConfigFactory.parse_file, cls._files),
).items()
}
@classmethod
def add_errors_file(cls, path: Union[Path, str]):
cls._files.append(path)
@classmethod
def generate_python_files(cls):
Generator(cls._apierrors_path / "errors", format_pep8=False).make_errors(
cls._get_codes()
)

96
apiserver/apierrors_generator/generator.py Normal file
View File

@ -0,0 +1,96 @@
import re
import json
import jinja2
import hashlib
from pathlib import Path
env = jinja2.Environment(
loader=jinja2.FileSystemLoader(str(Path(__file__).parent)),
autoescape=jinja2.select_autoescape(
disabled_extensions=("py",), default_for_string=False
),
trim_blocks=True,
lstrip_blocks=True,
)
def env_filter(name=None):
return lambda func: env.filters.setdefault(name or func.__name__, func)
@env_filter()
def cls_name(name):
delims = list(map(re.escape, (" ", "_")))
parts = re.split("|".join(delims), name)
return "".join(x.capitalize() for x in parts)
class Generator(object):
_base_class_name = "BaseError"
_base_class_module = "apiserver.apierrors.base"
def __init__(self, path, format_pep8=True, use_md5=True):
self._use_md5 = use_md5
self._format_pep8 = format_pep8
self._path = Path(path)
self._path.mkdir(parents=True, exist_ok=True)
def _make_init_file(self, path):
(self._path / path / "__init__.py").write_bytes(b"")
def _do_render(self, file, template, context):
with file.open("w") as f:
result = template.render(
base_class_name=self._base_class_name,
base_class_module=self._base_class_module,
**context
)
if self._format_pep8:
import autopep8
result = autopep8.fix_code(
result,
options={"aggressive": 1, "verbose": 0, "max_line_length": 120},
)
f.write(result)
def _make_section(self, name, code, subcodes):
self._do_render(
file=(self._path / name).with_suffix(".py"),
template=env.get_template("templates/section.jinja2"),
context=dict(code=code, subcodes=list(subcodes.items()),),
)
def _make_init(self, sections):
self._do_render(
file=(self._path / "__init__.py"),
template=env.get_template("templates/init.jinja2"),
context=dict(sections=sections,),
)
def _key_to_str(self, data):
if isinstance(data, dict):
return {str(k): self._key_to_str(v) for k, v in data.items()}
return data
def _calc_digest(self, data):
data = json.dumps(self._key_to_str(data), sort_keys=True)
return hashlib.md5(data.encode("utf8")).hexdigest()
def make_errors(self, errors):
digest = None
digest_file = self._path / "digest.md5"
if self._use_md5:
digest = self._calc_digest(errors)
if digest_file.is_file():
if digest_file.read_text() == digest:
return
self._make_init(errors)
for (code, section_name), subcodes in errors.items():
self._make_section(section_name, int(code), subcodes)
if self._use_md5:
digest_file.write_text(digest)

6
apiserver/apierrors_generator/templates/error.jinja2 Normal file
View File

@ -0,0 +1,6 @@
{% macro error_class(name, msg, code, subcode=0) %}
class {{ name }}({{ base_class_name }}):
_default_code = {{ code }}
_default_subcode = {{ subcode }}
_default_msg = "{{ msg|capitalize }}"
{% endmacro -%}

14
apiserver/apierrors_generator/templates/init.jinja2 Normal file
View File

@ -0,0 +1,14 @@
{% from 'templates/error.jinja2' import error_class with context %}
{% if sections %}
from {{ base_class_module }} import {{ base_class_name }}
{% endif %}
{% for _, name in sections %}
from . import {{ name }}
{% endfor %}
{% for code, name in sections %}
{{ error_class(name|cls_name, name|replace('_', ' '), code) }}
{% endfor %}

9
apiserver/apierrors_generator/templates/section.jinja2 Normal file
View File

@ -0,0 +1,9 @@
{% from 'templates/error.jinja2' import error_class with context %}
{% if subcodes %}
from {{ base_class_module }} import {{ base_class_name }}
{% endif %}
{% for subcode, (name, msg) in subcodes %}
{{ error_class(name|cls_name, msg, code, subcode|int) -}}
{% endfor %}