From 1cc6a8f7872a59365c706b255855805bb85424aa Mon Sep 17 00:00:00 2001 From: allegroai <> Date: Tue, 5 Jan 2021 18:11:22 +0200 Subject: [PATCH] Unify API model classes --- apiserver/apimodels/__init__.py | 198 +------------------------------ apiserver/apimodels/base.py | 203 +++++++++++++++++++++++++++++++- apiserver/database/fields.py | 20 +++- 3 files changed, 226 insertions(+), 195 deletions(-) diff --git a/apiserver/apimodels/__init__.py b/apiserver/apimodels/__init__.py index 4cb4d3f..6ac4e8c 100644 --- a/apiserver/apimodels/__init__.py +++ b/apiserver/apimodels/__init__.py @@ -1,109 +1,13 @@ from __future__ import absolute_import -from enum import Enum -from typing import Union, Type, Iterable +from textwrap import shorten -import jsonmodels.errors -import six -from jsonmodels import fields -from jsonmodels.fields import _LazyType, NotSet -from jsonmodels.models import Base as ModelBase -from jsonmodels.validators import Enum as EnumValidator from luqum.exceptions import ParseError from luqum.parser import parser from validators import email as email_validator, domain as domain_validator from apiserver.apierrors import errors -from apiserver.utilities.json import loads, dumps - - -def make_default(field_cls, default_value): - class _FieldWithDefault(field_cls): - def get_default_value(self): - return default_value - - return _FieldWithDefault - - -class ListField(fields.ListField): - def __init__(self, items_types=None, *args, default=NotSet, **kwargs): - if default is not NotSet and callable(default): - default = default() - - super(ListField, self).__init__(items_types, *args, default=default, **kwargs) - - def _cast_value(self, value): - try: - return super(ListField, self)._cast_value(value) - except TypeError: - return value - - def validate_single_value(self, item): - super(ListField, self).validate_single_value(item) - if isinstance(item, ModelBase): - item.validate() - - -class DictField(fields.BaseField): - types = (dict,) - - def __init__(self, value_types=None, *args, **kwargs): - self.value_types = self._assign_types(value_types) - super(DictField, self).__init__(*args, **kwargs) - - def get_default_value(self): - default = super(DictField, self).get_default_value() - if default is None and not self.required: - return {} - return default - - @staticmethod - def _assign_types(value_types): - if value_types: - try: - value_types = tuple(value_types) - except TypeError: - value_types = (value_types,) - else: - value_types = tuple() - - return tuple( - _LazyType(type_) if isinstance(type_, six.string_types) else type_ - for type_ in value_types - ) - - def validate(self, value): - super(DictField, self).validate(value) - - if not self.value_types: - return - - if not value: - return - - for item in value.values(): - self.validate_single_value(item) - - def validate_single_value(self, item): - if not self.value_types: - return - - if not isinstance(item, self.value_types): - raise jsonmodels.errors.ValidationError( - "All items must be instances " - 'of "{types}", and not "{type}".'.format( - types=", ".join([t.__name__ for t in self.value_types]), - type=type(item).__name__, - ) - ) - - -class IntField(fields.IntField): - def parse_value(self, value): - try: - return super(IntField, self).parse_value(value) - except (ValueError, TypeError): - return value +from .base import * def validate_lucene_query(value): @@ -112,7 +16,9 @@ def validate_lucene_query(value): try: parser.parse(value) except ParseError as e: - raise errors.bad_request.InvalidLuceneSyntax(error=e) + raise errors.bad_request.InvalidLuceneSyntax( + error=str(e), query=shorten(value, 50, placeholder="...") + ) class LuceneQueryField(fields.StringField): @@ -123,73 +29,6 @@ class LuceneQueryField(fields.StringField): validate_lucene_query(value) -class NullableEnumValidator(EnumValidator): - """Validator for enums that allows a None value.""" - - def validate(self, value): - if value is not None: - super(NullableEnumValidator, self).validate(value) - - -class EnumField(fields.StringField): - def __init__( - self, - values_or_type: Union[Iterable, Type[Enum]], - *args, - required=False, - default=None, - **kwargs - ): - choices = list(map(self.parse_value, values_or_type)) - validator_cls = EnumValidator if required else NullableEnumValidator - kwargs.setdefault("validators", []).append(validator_cls(*choices)) - super().__init__( - default=self.parse_value(default), required=required, *args, **kwargs - ) - - def parse_value(self, value): - if isinstance(value, Enum): - return str(value.value) - return super().parse_value(value) - - -class ActualEnumField(fields.StringField): - def __init__( - self, - enum_class: Type[Enum], - *args, - validators=None, - required=False, - default=None, - **kwargs - ): - self.__enum = enum_class - self.types = (enum_class,) - # noinspection PyTypeChecker - choices = list(enum_class) - validator_cls = EnumValidator if required else NullableEnumValidator - validators = [*(validators or []), validator_cls(*choices)] - super().__init__( - default=self.parse_value(default) if default else NotSet, - *args, - required=required, - validators=validators, - **kwargs - ) - - def parse_value(self, value): - if value is None and not self.required: - return self.get_default_value() - try: - # noinspection PyArgumentList - return self.__enum(value) - except ValueError: - return value - - def to_struct(self, value): - return super().to_struct(value.value) - - class EmailField(fields.StringField): def validate(self, value): super().validate(value) @@ -206,30 +45,3 @@ class DomainField(fields.StringField): return if domain_validator(value) is not True: raise errors.bad_request.InvalidDomainName() - - -class JsonSerializableMixin: - def to_json(self: ModelBase): - return dumps(self.to_struct()) - - @classmethod - def from_json(cls: Type[ModelBase], s): - return cls(**loads(s)) - - -def callable_default(cls: Type[fields.BaseField]) -> Type[fields.BaseField]: - class _Wrapped(cls): - _callable_default = None - - def get_default_value(self): - if self._callable_default: - return self._callable_default() - return super(_Wrapped, self).get_default_value() - - def __init__(self, *args, default=None, **kwargs): - if default and callable(default): - self._callable_default = default - default = default() - super(_Wrapped, self).__init__(*args, default=default, **kwargs) - - return _Wrapped diff --git a/apiserver/apimodels/base.py b/apiserver/apimodels/base.py index ca89dcd..900b7f7 100644 --- a/apiserver/apimodels/base.py +++ b/apiserver/apimodels/base.py @@ -1,8 +1,209 @@ +from __future__ import absolute_import + +from enum import Enum +from typing import Union, Type, Iterable + +import jsonmodels.errors +import six +from jsonmodels.fields import _LazyType, NotSet +from jsonmodels.models import Base as ModelBase +from jsonmodels.validators import Enum as EnumValidator + from jsonmodels import models, fields from jsonmodels.validators import Length from mongoengine.base import BaseDocument +from apiserver.utilities.json import loads, dumps -from apiserver.apimodels import DictField, ListField + +def make_default(field_cls, default_value): + class _FieldWithDefault(field_cls): + def get_default_value(self): + return default_value + + return _FieldWithDefault + + +class ListField(fields.ListField): + def __init__(self, items_types=None, *args, default=NotSet, **kwargs): + if default is not NotSet and callable(default): + default = default() + + super(ListField, self).__init__(items_types, *args, default=default, **kwargs) + + def _cast_value(self, value): + try: + return super(ListField, self)._cast_value(value) + except TypeError: + if len(self.items_types) == 1 and issubclass(self.items_types[0], Enum): + return self.items_types[0](value) + return value + + def validate_single_value(self, item): + super(ListField, self).validate_single_value(item) + if isinstance(item, ModelBase): + item.validate() + + +# since there is no distinction between None and empty DictField +# this value can be used as sentinel in order to distinguish +# between not set and empty DictField +DictFieldNotSet = {} + + +class DictField(fields.BaseField): + types = (dict,) + + def __init__(self, value_types=None, *args, **kwargs): + self.value_types = self._assign_types(value_types) + super(DictField, self).__init__(*args, **kwargs) + + def get_default_value(self): + default = super(DictField, self).get_default_value() + if default is None and not self.required: + return {} + return default + + @staticmethod + def _assign_types(value_types): + if value_types: + try: + value_types = tuple(value_types) + except TypeError: + value_types = (value_types,) + else: + value_types = tuple() + + return tuple( + _LazyType(type_) if isinstance(type_, six.string_types) else type_ + for type_ in value_types + ) + + def validate(self, value): + super(DictField, self).validate(value) + + if not self.value_types: + return + + if not value: + return + + for item in value.values(): + self.validate_single_value(item) + + def validate_single_value(self, item): + if not self.value_types: + return + + if not isinstance(item, self.value_types): + raise jsonmodels.errors.ValidationError( + "All items must be instances " + 'of "{types}", and not "{type}".'.format( + types=", ".join([t.__name__ for t in self.value_types]), + type=type(item).__name__, + ) + ) + + +class IntField(fields.IntField): + def parse_value(self, value): + try: + return super(IntField, self).parse_value(value) + except (ValueError, TypeError): + return value + + +class NullableEnumValidator(EnumValidator): + """Validator for enums that allows a None value.""" + + def validate(self, value): + if value is not None: + super(NullableEnumValidator, self).validate(value) + + +class EnumField(fields.StringField): + def __init__( + self, + values_or_type: Union[Iterable, Type[Enum]], + *args, + required=False, + default=None, + **kwargs + ): + choices = list(map(self.parse_value, values_or_type)) + validator_cls = EnumValidator if required else NullableEnumValidator + kwargs.setdefault("validators", []).append(validator_cls(*choices)) + super().__init__( + default=self.parse_value(default), required=required, *args, **kwargs + ) + + def parse_value(self, value): + if isinstance(value, Enum): + return str(value.value) + return super().parse_value(value) + + +class ActualEnumField(fields.StringField): + def __init__( + self, + enum_class: Type[Enum], + *args, + validators=None, + required=False, + default=None, + **kwargs + ): + self.__enum = enum_class + self.types = (enum_class,) + # noinspection PyTypeChecker + choices = list(enum_class) + validator_cls = EnumValidator if required else NullableEnumValidator + validators = [*(validators or []), validator_cls(*choices)] + super().__init__( + default=self.parse_value(default) if default else NotSet, + *args, + required=required, + validators=validators, + **kwargs + ) + + def parse_value(self, value): + if value is None and not self.required: + return self.get_default_value() + try: + # noinspection PyArgumentList + return self.__enum(value) + except ValueError: + return value + + def to_struct(self, value): + return super().to_struct(value.value) + + +class JsonSerializableMixin: + def to_json(self: ModelBase): + return dumps(self.to_struct()) + + @classmethod + def from_json(cls: Type[ModelBase], s): + return cls(**loads(s)) + + +def callable_default(cls: Type[fields.BaseField]) -> Type[fields.BaseField]: + class _Wrapped(cls): + _callable_default = None + + def get_default_value(self): + if self._callable_default: + return self._callable_default() + return super(_Wrapped, self).get_default_value() + + def __init__(self, *args, default=None, **kwargs): + if default and callable(default): + self._callable_default = default + default = default() + super(_Wrapped, self).__init__(*args, default=default, **kwargs) + + return _Wrapped class MongoengineFieldsDict(DictField): diff --git a/apiserver/database/fields.py b/apiserver/database/fields.py index ca3ee9b..52e7575 100644 --- a/apiserver/database/fields.py +++ b/apiserver/database/fields.py @@ -14,7 +14,7 @@ from mongoengine import ( DictField, DynamicField, ) -from mongoengine.fields import key_not_string, key_starts_with_dollar +from mongoengine.fields import key_not_string, key_starts_with_dollar, EmailField NoneType = type(None) @@ -93,6 +93,24 @@ class CustomFloatField(FloatField): self.error("Float value must be greater than %s" % str(self.greater_than)) +class CanonicEmailField(EmailField): + """email field that is always lower cased""" + def __set__(self, instance, value: str): + if value is not None: + try: + value = value.lower() + except AttributeError: + pass + super().__set__(instance, value) + + def prepare_query_value(self, op, value): + if not isinstance(op, six.string_types): + return value + if value is not None: + value = value.lower() + return super().prepare_query_value(op, value) + + class StrippedStringField(StringField): def __init__( self, regex=None, max_length=None, min_length=None, strip_chars=None, **kwargs