ClearML/clearml
1
0
Fork 0
mirror of https://github.com/clearml/clearml synced 2025-04-22 23:35:03 +00:00
Code Issues Actions Packages Projects Releases Wiki Activity

Add exception guard

Browse Source
This commit is contained in:
allegroai 2019-07-08 23:27:53 +03:00
parent 02ba0e28d5
commit 8a1e31c4f0
2 changed files with 32 additions and 24 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

6
trains/backend_interface/base.py
View File

@ -134,7 +134,11 @@ class IdObjectBase(InterfaceBase):
def reload(self): def reload(self):
if not self.id: if not self.id:
raise ValueError('Failed reloading %s: missing id' % type(self).__name__) raise ValueError('Failed reloading %s: missing id' % type(self).__name__)
self._data = self._reload() # noinspection PyBroadException
try:
self._data = self._reload()
except Exception:
pass
@classmethod @classmethod
def normalize_id(cls, id): def normalize_id(cls, id):

50
trains/backend_interface/task/development/stop_signal.py
View File

@ -28,29 +28,33 @@ class TaskStopSignal(object):
self._task_reset_state_counter = 0 self._task_reset_state_counter = 0
def test(self): def test(self):
status = self.task.status # noinspection PyBroadException
message = self.task.data.status_message try:
status = self.task.status
message = self.task.data.status_message
if status == tasks.TaskStatusEnum.in_progress and "stopping" in message: if status == tasks.TaskStatusEnum.in_progress and "stopping" in message:
return TaskStopReason.stopped return TaskStopReason.stopped
_expected_statuses = ( _expected_statuses = (
tasks.TaskStatusEnum.created, tasks.TaskStatusEnum.created,
tasks.TaskStatusEnum.queued, tasks.TaskStatusEnum.queued,
tasks.TaskStatusEnum.in_progress, tasks.TaskStatusEnum.in_progress,
)
if status not in _expected_statuses and "worker" not in message:
return TaskStopReason.status_changed
if status == tasks.TaskStatusEnum.created:
self._task_reset_state_counter += 1
if self._task_reset_state_counter >= self._number_of_consecutive_reset_tests:
return TaskStopReason.reset
self.task.get_logger().warning(
"Task {} was reset! if state is consistent we shall terminate.".format(self.task.id),
) )
else:
self._task_reset_state_counter = 0 if status not in _expected_statuses and "worker" not in message:
return TaskStopReason.status_changed
if status == tasks.TaskStatusEnum.created:
self._task_reset_state_counter += 1
if self._task_reset_state_counter >= self._number_of_consecutive_reset_tests:
return TaskStopReason.reset
self.task.get_logger().warning(
"Task {} was reset! if state is consistent we shall terminate.".format(self.task.id),
)
else:
self._task_reset_state_counter = 0
except Exception:
return None