Added exclude support when converting mongo objects to dictionary

apiserver/database/model/base.py

@@ -26,7 +26,7 @@ from apiserver.bll.redis_cache_manager import RedisCacheManager
 from apiserver.config_repo import config
 from apiserver.database import Database
 from apiserver.database.errors import MakeGetAllQueryError
-from apiserver.database.projection import project_dict, ProjectionHelper
+from apiserver.database.projection import ProjectionHelper
 from apiserver.database.props import PropsMixin
 from apiserver.database.query import RegexQ, RegexWrapper, RegexQCombination
 from apiserver.database.utils import (
@@ -36,6 +36,7 @@ from apiserver.database.utils import (
     field_exists,
 )
 from apiserver.redis_manager import redman
+from apiserver.utilities.dicts import project_dict, exclude_fields_from_dict
 
 log = config.logger("dbmodel")
 
@@ -55,17 +56,25 @@ class ProperDictMixin(object):
         strip_private=True,
         only=None,
         extra_dict=None,
+        exclude=None,
     ) -> dict:
         return self.properize_dict(
             self.to_mongo(use_db_field=False).to_dict(),
             strip_private=strip_private,
             only=only,
             extra_dict=extra_dict,
+            exclude=exclude,
         )
 
     @classmethod
     def properize_dict(
-        cls, d, strip_private=True, only=None, extra_dict=None, normalize_id=True
+        cls,
+        d,
+        strip_private=True,
+        only=None,
+        extra_dict=None,
+        exclude=None,
+        normalize_id=True,
     ):
         res = d
         if normalize_id and "_id" in res:
@@ -76,6 +85,9 @@ class ProperDictMixin(object):
             res = project_dict(res, only)
         if extra_dict:
             res.update(extra_dict)
+        if exclude:
+            exclude_fields_from_dict(res, exclude)
+
         return res
 
 
@@ -385,7 +397,11 @@ class GetMixin(PropsMixin):
                                 value = parse_datetime(m.group("value"))
                                 prefix = m.group("prefix")
                                 modifier = ACCESS_MODIFIER.get(prefix)
-                                f = field if not modifier else "__".join((field, modifier))
+                                f = (
+                                    field
+                                    if not modifier
+                                    else "__".join((field, modifier))
+                                )
                                 dict_query[f] = value
                             except (ValueError, OverflowError):
                                 pass
@@ -1000,7 +1016,11 @@ class GetMixin(PropsMixin):
             query_sets = [qs.fields(**projection_fields) for qs in query_sets]
 
         if start is None or not size:
-            return [obj.to_proper_dict(only=include) for qs in query_sets for obj in qs]
+            return [
+                obj.to_proper_dict(only=include, exclude=exclude)
+                for qs in query_sets
+                for obj in qs
+            ]
 
         # add paging
         ret = []
@@ -1008,7 +1028,7 @@ class GetMixin(PropsMixin):
         for i, qs in enumerate(query_sets):
             last_size = len(ret)
             ret.extend(
-                obj.to_proper_dict(only=include)
+                obj.to_proper_dict(only=include, exclude=exclude)
                 for obj in (qs.skip(start) if start else qs).limit(size)
             )
             added = len(ret) - last_size

apiserver/database/projection.py

@@ -9,65 +9,6 @@ from apiserver.database.props import PropsMixin
 SEP = "."
 
 
-def project_dict(data, projection, separator=SEP):
-    """
-    Project partial data from a dictionary into a new dictionary
-    :param data: Input dictionary
-    :param projection: List of dictionary paths (each a string with field names separated using a separator)
-    :param separator: Separator (default is '.')
-    :return: A new dictionary containing only the projected parts from the original dictionary
-    """
-    assert isinstance(data, dict)
-    result = {}
-
-    def copy_path(path_parts, source, destination):
-        src, dst = source, destination
-        try:
-            for depth, path_part in enumerate(path_parts[:-1]):
-                src_part = src[path_part]
-                if isinstance(src_part, dict):
-                    src = src_part
-                    dst = dst.setdefault(path_part, {})
-                elif isinstance(src_part, (list, tuple)):
-                    if path_part not in dst:
-                        dst[path_part] = [{} for _ in range(len(src_part))]
-                    elif not isinstance(dst[path_part], (list, tuple)):
-                        raise TypeError(
-                            "Incompatible destination type %s for %s (list expected)"
-                            % (type(dst), separator.join(path_parts[: depth + 1]))
-                        )
-                    elif not len(dst[path_part]) == len(src_part):
-                        raise ValueError(
-                            "Destination list length differs from source length for %s"
-                            % separator.join(path_parts[: depth + 1])
-                        )
-
-                    dst[path_part] = [
-                        copy_path(path_parts[depth + 1 :], s, d)
-                        for s, d in zip(src_part, dst[path_part])
-                    ]
-
-                    return destination
-                else:
-                    raise TypeError(
-                        "Unsupported projection type %s for %s"
-                        % (type(src), separator.join(path_parts[: depth + 1]))
-                    )
-
-            last_part = path_parts[-1]
-            dst[last_part] = src[last_part]
-        except KeyError:
-            # Projection field not in source, no biggie.
-            pass
-        return destination
-
-    for projection_path in sorted(projection):
-        copy_path(
-            path_parts=projection_path.split(separator), source=data, destination=result
-        )
-    return result
-
-
 class _ReferenceProxy(dict):
     def __init__(self, id):
         super(_ReferenceProxy, self).__init__(**({"id": id} if id else {}))

apiserver/utilities/dicts.py

@@ -79,3 +79,75 @@ def nested_set(dictionary: dict, path: Union[Sequence[str], str], value: Any):
         node = node.get(key)
 
     node[last_key] = value
+
+
+def exclude_fields_from_dict(data: dict, fields: Sequence[str], separator="."):
+    """
+    Performs in place fields exclusion on the passed dict
+    """
+    assert isinstance(data, dict)
+    if not fields:
+        return
+
+    exclude_paths = [e.split(separator) for e in fields]
+    for path in sorted(exclude_paths):
+        nested_delete(data, path)
+
+
+def project_dict(data: dict, projection: Sequence[str], separator=".") -> dict:
+    """
+    Project partial data from a dictionary into a new dictionary
+    :param data: Input dictionary
+    :param projection: List of dictionary paths (each a string with field names separated using a separator)
+    :param separator: Separator (default is '.')
+    :return: A new dictionary containing only the projected parts from the original dictionary
+    """
+    assert isinstance(data, dict)
+    result = {}
+
+    def copy_path(path_parts, source, destination):
+        src, dst = source, destination
+        try:
+            for depth, path_part in enumerate(path_parts[:-1]):
+                src_part = src[path_part]
+                if isinstance(src_part, dict):
+                    src = src_part
+                    dst = dst.setdefault(path_part, {})
+                elif isinstance(src_part, (list, tuple)):
+                    if path_part not in dst:
+                        dst[path_part] = [{} for _ in range(len(src_part))]
+                    elif not isinstance(dst[path_part], (list, tuple)):
+                        raise TypeError(
+                            "Incompatible destination type %s for %s (list expected)"
+                            % (type(dst), separator.join(path_parts[: depth + 1]))
+                        )
+                    elif not len(dst[path_part]) == len(src_part):
+                        raise ValueError(
+                            "Destination list length differs from source length for %s"
+                            % separator.join(path_parts[: depth + 1])
+                        )
+
+                    dst[path_part] = [
+                        copy_path(path_parts[depth + 1 :], s, d)
+                        for s, d in zip(src_part, dst[path_part])
+                    ]
+
+                    return destination
+                else:
+                    raise TypeError(
+                        "Unsupported projection type %s for %s"
+                        % (type(src), separator.join(path_parts[: depth + 1]))
+                    )
+
+            last_part = path_parts[-1]
+            dst[last_part] = src[last_part]
+        except KeyError:
+            # Projection field not in source, no biggie.
+            pass
+        return destination
+
+    for projection_path in sorted(projection):
+        copy_path(
+            path_parts=projection_path.split(separator), source=data, destination=result
+        )
+    return result