ClearML/clearml-server
1
0
Fork 0
mirror of https://github.com/clearml/clearml-server synced 2025-01-31 10:56:48 +00:00
Code Issues Actions Packages Projects Releases Wiki Activity

Allow projection of more than 500 items

Browse Source
This commit is contained in:
allegroai 2023-07-26 18:46:58 +03:00
parent 4c98b87554
commit 1e51e2e221
1 changed files with 9 additions and 4 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

7
apiserver/database/projection.py
View File

@ -3,10 +3,14 @@ from concurrent.futures import ThreadPoolExecutor
from itertools import groupby, chain from itertools import groupby, chain
from typing import Sequence, Dict, Callable from typing import Sequence, Dict, Callable
from boltons import iterutils
from apiserver.apierrors import errors from apiserver.apierrors import errors
from apiserver.config_repo import config
from apiserver.database.props import PropsMixin from apiserver.database.props import PropsMixin
SEP = "." SEP = "."
max_items_per_fetch = config.get("services._mongo.max_page_size", 500)
class _ReferenceProxy(dict): class _ReferenceProxy(dict):
@ -278,8 +282,9 @@ class ProjectionHelper(object):
doc_only = list(filter(None, data["only"])) doc_only = list(filter(None, data["only"]))
doc_only = list({"id"} | set(doc_only)) if doc_only else None doc_only = list({"id"} | set(doc_only)) if doc_only else None
for ids_chunk in iterutils.chunked_iter(ids, max_items_per_fetch):
for res in projection_func( for res in projection_func(
doc_type=doc_type, projection=doc_only, ids=ids doc_type=doc_type, projection=doc_only, ids=ids_chunk
): ):
self._proxy_manager.update(res) self._proxy_manager.update(res)