ClearML/clearml
1
0
Fork 0
mirror of https://github.com/clearml/clearml synced 2025-06-26 18:16:07 +00:00
Code Issues Actions Packages Projects Releases Wiki Activity

Optimize task refresh while pulling task status in local worker and last iteration for Resource Monitoring

Browse Source
This commit is contained in:
allegroai
2020-03-05 11:40:27 +02:00
parent cc1508b2bd
commit e3ae4f4e26
3 changed files with 30 additions and 3 deletions
Download Patch File Download Diff File Expand all files Collapse all files

13
trains/backend_interface/task/development/stop_signal.py
View File

@@ -30,10 +30,15 @@ class TaskStopSignal(object):
def test(self):
# noinspection PyBroadException
try:
status = str(self.task.status)
message = self.task.data.status_message
# we use internal status read, so we do not need to constantly pull the entire task object,
# it might be large, and we want to also avoid the edit lock on it.
status, message = self.task._get_status()
status = str(status)
message = str(message)
if status == str(tasks.TaskStatusEnum.in_progress) and "stopping" in message:
# make sure we syn the entire task object
self.task.reload()
return TaskStopReason.stopped
_expected_statuses = (
@@ -43,12 +48,16 @@ class TaskStopSignal(object):
)
if status not in _expected_statuses and "worker" not in message:
# make sure we syn the entire task object
self.task.reload()
return TaskStopReason.status_changed
if status == str(tasks.TaskStatusEnum.created):
self._task_reset_state_counter += 1
if self._task_reset_state_counter >= self._number_of_consecutive_reset_tests:
# make sure we syn the entire task object
self.task.reload()
return TaskStopReason.reset
self.task.log.warning(