Add debug.ping endpoint

Optimize exhausted scrolls by using a fixed empty scroll

server/bll/event/event_bll.py

@@ -32,6 +32,7 @@ LOCKED_TASK_STATUSES = (TaskStatus.publishing, TaskStatus.published)
 
 class EventBLL(object):
     id_fields = ("task", "iter", "metric", "variant", "key")
+    empty_scroll = "FFFF"
 
     def __init__(self, events_es=None, redis=None):
         self.es = events_es or es_factory.connect("events")
@@ -321,6 +322,9 @@ class EventBLL(object):
         batch_size=10000,
         scroll_id=None,
     ):
+        if scroll_id == self.empty_scroll:
+            return [], scroll_id, 0
+
         if scroll_id:
             with translate_errors_context(), TimingContext("es", "task_log_events"):
                 es_res = self.es.scroll(scroll_id=scroll_id, scroll="1h")
@@ -407,6 +411,9 @@ class EventBLL(object):
         size: int = 500,
         scroll_id: str = None,
     ):
+        if scroll_id == self.empty_scroll:
+            return [], scroll_id, 0
+
         if scroll_id:
             with translate_errors_context(), TimingContext("es", "get_task_events"):
                 es_res = self.es.scroll(scroll_id=scroll_id, scroll="1h")
@@ -474,7 +481,7 @@ class EventBLL(object):
         next_scroll_id = es_res.get("_scroll_id")
         if next_scroll_id and not events:
             self.es.clear_scroll(scroll_id=next_scroll_id)
-            next_scroll_id = None
+            next_scroll_id = self.empty_scroll
 
         return events, total_events, next_scroll_id
 
@@ -490,6 +497,8 @@ class EventBLL(object):
         size=500,
         scroll_id=None,
     ):
+        if scroll_id == self.empty_scroll:
+            return [], scroll_id, 0
 
         if scroll_id:
             with translate_errors_context(), TimingContext("es", "get_task_events"):

server/bll/statistics/stats_reporter.py

@@ -19,7 +19,7 @@ from config.info import get_deployment_type
 from database.model import Company, User
 from database.model.queue import Queue
 from database.model.task.task import Task
-from utilities import safe_get
+from tools import safe_get
 from utilities.json import dumps
 from utilities.threads_manager import ThreadsManager
 from version import __version__ as current_version

server/schema/services/debug.conf

@@ -0,0 +1,16 @@
+_description: "debugging utilities"
+ping {
+    authorize: false
+    "2.9" {
+        description: "Ping server"
+        request {
+            type: object
+            additionalProperties: true
+        }
+        response {
+            type: object
+            properties: {
+            }
+        }
+    }
+}

server/service_repo/__init__.py

@@ -8,7 +8,7 @@ from .endpoint import EndpointFunc, Endpoint
 from .service_repo import ServiceRepo
 
 
-__all__ = ["endpoint"]
+__all__ = ["APICall", "endpoint"]
 
 
 LegacyEndpointFunc = Callable[[APICall], None]

server/services/debug.py

@@ -0,0 +1,6 @@
+from service_repo import APICall, endpoint
+
+
+@endpoint("debug.ping")
+def ping(call: APICall, _, __):
+    call.result.data = {"msg": "Because it trains cats and dogs"}

server/utilities/__init__.py

@@ -1,12 +1,2 @@
-import dpath
-
-
 def strict_map(*args, **kwargs):
     return list(map(*args, **kwargs))
-
-
-def safe_get(obj, glob, default=None):
-    try:
-        return dpath.get(obj, glob)
-    except KeyError:
-        return default