Add exception guard

trains/backend_interface/base.py

@@ -134,7 +134,11 @@ class IdObjectBase(InterfaceBase):
     def reload(self):
         if not self.id:
             raise ValueError('Failed reloading %s: missing id' % type(self).__name__)
-        self._data = self._reload()
+        # noinspection PyBroadException
+        try:
+            self._data = self._reload()
+        except Exception:
+            pass
 
     @classmethod
     def normalize_id(cls, id):

trains/backend_interface/task/development/stop_signal.py

@@ -28,29 +28,33 @@ class TaskStopSignal(object):
         self._task_reset_state_counter = 0
 
     def test(self):
-        status = self.task.status
-        message = self.task.data.status_message
+        # noinspection PyBroadException
+        try:
+            status = self.task.status
+            message = self.task.data.status_message
 
-        if status == tasks.TaskStatusEnum.in_progress and "stopping" in message:
-            return TaskStopReason.stopped
+            if status == tasks.TaskStatusEnum.in_progress and "stopping" in message:
+                return TaskStopReason.stopped
 
-        _expected_statuses = (
-            tasks.TaskStatusEnum.created,
-            tasks.TaskStatusEnum.queued,
-            tasks.TaskStatusEnum.in_progress,
-        )
-
-        if status not in _expected_statuses and "worker" not in message:
-            return TaskStopReason.status_changed
-
-        if status == tasks.TaskStatusEnum.created:
-            self._task_reset_state_counter += 1
-
-            if self._task_reset_state_counter >= self._number_of_consecutive_reset_tests:
-                return TaskStopReason.reset
-
-            self.task.get_logger().warning(
-                "Task {} was reset! if state is consistent we shall terminate.".format(self.task.id),
+            _expected_statuses = (
+                tasks.TaskStatusEnum.created,
+                tasks.TaskStatusEnum.queued,
+                tasks.TaskStatusEnum.in_progress,
             )
-        else:
-            self._task_reset_state_counter = 0
+
+            if status not in _expected_statuses and "worker" not in message:
+                return TaskStopReason.status_changed
+
+            if status == tasks.TaskStatusEnum.created:
+                self._task_reset_state_counter += 1
+
+                if self._task_reset_state_counter >= self._number_of_consecutive_reset_tests:
+                    return TaskStopReason.reset
+
+                self.task.get_logger().warning(
+                    "Task {} was reset! if state is consistent we shall terminate.".format(self.task.id),
+                )
+            else:
+                self._task_reset_state_counter = 0
+        except Exception:
+            return None