From 58d826e427ab9e53da5d2b9c4b6912fb3520dec4 Mon Sep 17 00:00:00 2001
From: allegroai <>
Date: Sat, 23 Sep 2023 17:36:01 +0300
Subject: [PATCH] Fail-safe Kafka pulling

---
 clearml_serving/statistics/metrics.py | 11 ++++++++++-
 1 file changed, 10 insertions(+), 1 deletion(-)

diff --git a/clearml_serving/statistics/metrics.py b/clearml_serving/statistics/metrics.py
index afdd095..6ef0878 100644
--- a/clearml_serving/statistics/metrics.py
+++ b/clearml_serving/statistics/metrics.py
@@ -240,13 +240,22 @@ class StatisticsController(object):
                 sleep(30)
 
         # we will never leave this loop
-        for message in consumer:
+        while True:
+            # noinspection PyBroadException
+            try:
+                message = next(consumer)
+            except Exception:
+                print("Warning: failed to pull kafka consumer pipe")
+                sleep(5)
+                continue
+
             # noinspection PyBroadException
             try:
                 list_data = json.loads(message.value.decode("utf-8"))
             except Exception:
                 print("Warning: failed to decode kafka stats message")
                 continue
+
             for data in list_data:
                 try:
                     url = data.pop("_url", None)