From 8a1e31c4f085b8c664ab406b7b38d8c38b901fa8 Mon Sep 17 00:00:00 2001
From: allegroai <>
Date: Mon, 8 Jul 2019 23:27:53 +0300
Subject: [PATCH] Add exception guard

---
 trains/backend_interface/base.py              |  6 ++-
 .../task/development/stop_signal.py           | 50 ++++++++++---------
 2 files changed, 32 insertions(+), 24 deletions(-)

diff --git a/trains/backend_interface/base.py b/trains/backend_interface/base.py
index 1a16f9d3..337de106 100644
--- a/trains/backend_interface/base.py
+++ b/trains/backend_interface/base.py
@@ -134,7 +134,11 @@ class IdObjectBase(InterfaceBase):
     def reload(self):
         if not self.id:
             raise ValueError('Failed reloading %s: missing id' % type(self).__name__)
-        self._data = self._reload()
+        # noinspection PyBroadException
+        try:
+            self._data = self._reload()
+        except Exception:
+            pass
 
     @classmethod
     def normalize_id(cls, id):
diff --git a/trains/backend_interface/task/development/stop_signal.py b/trains/backend_interface/task/development/stop_signal.py
index 40e7c6cb..24f146a2 100644
--- a/trains/backend_interface/task/development/stop_signal.py
+++ b/trains/backend_interface/task/development/stop_signal.py
@@ -28,29 +28,33 @@ class TaskStopSignal(object):
         self._task_reset_state_counter = 0
 
     def test(self):
-        status = self.task.status
-        message = self.task.data.status_message
+        # noinspection PyBroadException
+        try:
+            status = self.task.status
+            message = self.task.data.status_message
 
-        if status == tasks.TaskStatusEnum.in_progress and "stopping" in message:
-            return TaskStopReason.stopped
+            if status == tasks.TaskStatusEnum.in_progress and "stopping" in message:
+                return TaskStopReason.stopped
 
-        _expected_statuses = (
-            tasks.TaskStatusEnum.created,
-            tasks.TaskStatusEnum.queued,
-            tasks.TaskStatusEnum.in_progress,
-        )
-
-        if status not in _expected_statuses and "worker" not in message:
-            return TaskStopReason.status_changed
-
-        if status == tasks.TaskStatusEnum.created:
-            self._task_reset_state_counter += 1
-
-            if self._task_reset_state_counter >= self._number_of_consecutive_reset_tests:
-                return TaskStopReason.reset
-
-            self.task.get_logger().warning(
-                "Task {} was reset! if state is consistent we shall terminate.".format(self.task.id),
+            _expected_statuses = (
+                tasks.TaskStatusEnum.created,
+                tasks.TaskStatusEnum.queued,
+                tasks.TaskStatusEnum.in_progress,
             )
-        else:
-            self._task_reset_state_counter = 0
+
+            if status not in _expected_statuses and "worker" not in message:
+                return TaskStopReason.status_changed
+
+            if status == tasks.TaskStatusEnum.created:
+                self._task_reset_state_counter += 1
+
+                if self._task_reset_state_counter >= self._number_of_consecutive_reset_tests:
+                    return TaskStopReason.reset
+
+                self.task.get_logger().warning(
+                    "Task {} was reset! if state is consistent we shall terminate.".format(self.task.id),
+                )
+            else:
+                self._task_reset_state_counter = 0
+        except Exception:
+            return None