From 98ed3075dd330b0d10306d89b856240ed758d33d Mon Sep 17 00:00:00 2001 From: allegroai <> Date: Thu, 29 Sep 2022 19:21:28 +0300 Subject: [PATCH] Added exclude support when converting mongo objects to dictionary --- apiserver/database/model/base.py | 30 ++++++++++--- apiserver/database/projection.py | 59 -------------------------- apiserver/utilities/dicts.py | 72 ++++++++++++++++++++++++++++++++ 3 files changed, 97 insertions(+), 64 deletions(-) diff --git a/apiserver/database/model/base.py b/apiserver/database/model/base.py index 71cbaf8..ef654c4 100644 --- a/apiserver/database/model/base.py +++ b/apiserver/database/model/base.py @@ -26,7 +26,7 @@ from apiserver.bll.redis_cache_manager import RedisCacheManager from apiserver.config_repo import config from apiserver.database import Database from apiserver.database.errors import MakeGetAllQueryError -from apiserver.database.projection import project_dict, ProjectionHelper +from apiserver.database.projection import ProjectionHelper from apiserver.database.props import PropsMixin from apiserver.database.query import RegexQ, RegexWrapper, RegexQCombination from apiserver.database.utils import ( @@ -36,6 +36,7 @@ from apiserver.database.utils import ( field_exists, ) from apiserver.redis_manager import redman +from apiserver.utilities.dicts import project_dict, exclude_fields_from_dict log = config.logger("dbmodel") @@ -55,17 +56,25 @@ class ProperDictMixin(object): strip_private=True, only=None, extra_dict=None, + exclude=None, ) -> dict: return self.properize_dict( self.to_mongo(use_db_field=False).to_dict(), strip_private=strip_private, only=only, extra_dict=extra_dict, + exclude=exclude, ) @classmethod def properize_dict( - cls, d, strip_private=True, only=None, extra_dict=None, normalize_id=True + cls, + d, + strip_private=True, + only=None, + extra_dict=None, + exclude=None, + normalize_id=True, ): res = d if normalize_id and "_id" in res: @@ -76,6 +85,9 @@ class ProperDictMixin(object): res = project_dict(res, only) if extra_dict: res.update(extra_dict) + if exclude: + exclude_fields_from_dict(res, exclude) + return res @@ -385,7 +397,11 @@ class GetMixin(PropsMixin): value = parse_datetime(m.group("value")) prefix = m.group("prefix") modifier = ACCESS_MODIFIER.get(prefix) - f = field if not modifier else "__".join((field, modifier)) + f = ( + field + if not modifier + else "__".join((field, modifier)) + ) dict_query[f] = value except (ValueError, OverflowError): pass @@ -1000,7 +1016,11 @@ class GetMixin(PropsMixin): query_sets = [qs.fields(**projection_fields) for qs in query_sets] if start is None or not size: - return [obj.to_proper_dict(only=include) for qs in query_sets for obj in qs] + return [ + obj.to_proper_dict(only=include, exclude=exclude) + for qs in query_sets + for obj in qs + ] # add paging ret = [] @@ -1008,7 +1028,7 @@ class GetMixin(PropsMixin): for i, qs in enumerate(query_sets): last_size = len(ret) ret.extend( - obj.to_proper_dict(only=include) + obj.to_proper_dict(only=include, exclude=exclude) for obj in (qs.skip(start) if start else qs).limit(size) ) added = len(ret) - last_size diff --git a/apiserver/database/projection.py b/apiserver/database/projection.py index 0ab7961..b74f238 100644 --- a/apiserver/database/projection.py +++ b/apiserver/database/projection.py @@ -9,65 +9,6 @@ from apiserver.database.props import PropsMixin SEP = "." -def project_dict(data, projection, separator=SEP): - """ - Project partial data from a dictionary into a new dictionary - :param data: Input dictionary - :param projection: List of dictionary paths (each a string with field names separated using a separator) - :param separator: Separator (default is '.') - :return: A new dictionary containing only the projected parts from the original dictionary - """ - assert isinstance(data, dict) - result = {} - - def copy_path(path_parts, source, destination): - src, dst = source, destination - try: - for depth, path_part in enumerate(path_parts[:-1]): - src_part = src[path_part] - if isinstance(src_part, dict): - src = src_part - dst = dst.setdefault(path_part, {}) - elif isinstance(src_part, (list, tuple)): - if path_part not in dst: - dst[path_part] = [{} for _ in range(len(src_part))] - elif not isinstance(dst[path_part], (list, tuple)): - raise TypeError( - "Incompatible destination type %s for %s (list expected)" - % (type(dst), separator.join(path_parts[: depth + 1])) - ) - elif not len(dst[path_part]) == len(src_part): - raise ValueError( - "Destination list length differs from source length for %s" - % separator.join(path_parts[: depth + 1]) - ) - - dst[path_part] = [ - copy_path(path_parts[depth + 1 :], s, d) - for s, d in zip(src_part, dst[path_part]) - ] - - return destination - else: - raise TypeError( - "Unsupported projection type %s for %s" - % (type(src), separator.join(path_parts[: depth + 1])) - ) - - last_part = path_parts[-1] - dst[last_part] = src[last_part] - except KeyError: - # Projection field not in source, no biggie. - pass - return destination - - for projection_path in sorted(projection): - copy_path( - path_parts=projection_path.split(separator), source=data, destination=result - ) - return result - - class _ReferenceProxy(dict): def __init__(self, id): super(_ReferenceProxy, self).__init__(**({"id": id} if id else {})) diff --git a/apiserver/utilities/dicts.py b/apiserver/utilities/dicts.py index 8d6cc2d..3850f6e 100644 --- a/apiserver/utilities/dicts.py +++ b/apiserver/utilities/dicts.py @@ -79,3 +79,75 @@ def nested_set(dictionary: dict, path: Union[Sequence[str], str], value: Any): node = node.get(key) node[last_key] = value + + +def exclude_fields_from_dict(data: dict, fields: Sequence[str], separator="."): + """ + Performs in place fields exclusion on the passed dict + """ + assert isinstance(data, dict) + if not fields: + return + + exclude_paths = [e.split(separator) for e in fields] + for path in sorted(exclude_paths): + nested_delete(data, path) + + +def project_dict(data: dict, projection: Sequence[str], separator=".") -> dict: + """ + Project partial data from a dictionary into a new dictionary + :param data: Input dictionary + :param projection: List of dictionary paths (each a string with field names separated using a separator) + :param separator: Separator (default is '.') + :return: A new dictionary containing only the projected parts from the original dictionary + """ + assert isinstance(data, dict) + result = {} + + def copy_path(path_parts, source, destination): + src, dst = source, destination + try: + for depth, path_part in enumerate(path_parts[:-1]): + src_part = src[path_part] + if isinstance(src_part, dict): + src = src_part + dst = dst.setdefault(path_part, {}) + elif isinstance(src_part, (list, tuple)): + if path_part not in dst: + dst[path_part] = [{} for _ in range(len(src_part))] + elif not isinstance(dst[path_part], (list, tuple)): + raise TypeError( + "Incompatible destination type %s for %s (list expected)" + % (type(dst), separator.join(path_parts[: depth + 1])) + ) + elif not len(dst[path_part]) == len(src_part): + raise ValueError( + "Destination list length differs from source length for %s" + % separator.join(path_parts[: depth + 1]) + ) + + dst[path_part] = [ + copy_path(path_parts[depth + 1 :], s, d) + for s, d in zip(src_part, dst[path_part]) + ] + + return destination + else: + raise TypeError( + "Unsupported projection type %s for %s" + % (type(src), separator.join(path_parts[: depth + 1])) + ) + + last_part = path_parts[-1] + dst[last_part] = src[last_part] + except KeyError: + # Projection field not in source, no biggie. + pass + return destination + + for projection_path in sorted(projection): + copy_path( + path_parts=projection_path.split(separator), source=data, destination=result + ) + return result