Fix update project time on task changes

Fix project time in non responsive tasks watchdog
This commit is contained in:
allegroai
2021-01-05 18:19:45 +02:00
parent 59994ccf9c
commit e2deff4eef
5 changed files with 53 additions and 31 deletions

View File

@@ -1,8 +1,7 @@
from datetime import timedelta, datetime
from time import sleep
from apiserver.apierrors import errors
from apiserver.bll.task import ChangeStatusRequest
from apiserver.bll.task import update_project_time
from apiserver.config_repo import config
from apiserver.database.model.task.task import TaskStatus, Task
from apiserver.utilities.threads_manager import ThreadsManager
@@ -71,19 +70,29 @@ class NonResponsiveTasksWatchdog:
return 0
err_count = 0
project_ids = set()
now = datetime.utcnow()
for task in tasks:
log.info(
f"Stopping {task.id} ({task.name}), last updated at {task.last_update}"
)
# noinspection PyBroadException
try:
ChangeStatusRequest(
task=task,
new_status=TaskStatus.stopped,
updated = Task.objects(id=task.id, status=task.status).update(
status=TaskStatus.stopped,
status_reason="Forced stop (non-responsive)",
status_message="Forced stop (non-responsive)",
force=True,
).execute()
except errors.bad_request.FailedChangingTaskStatus:
err_count += 1
status_changed=now,
last_update=now,
last_change=now,
)
if updated:
project_ids.add(task.project)
else:
err_count += 1
except Exception as ex:
log.error("Failed setting status: %s", str(ex))
update_project_time(list(project_ids))
return len(tasks) - err_count

View File

@@ -34,7 +34,7 @@ from apiserver.timing_context import TimingContext
from apiserver.utilities.parameter_key_escaper import ParameterKeyEscaper
from .artifacts import artifacts_prepare_for_save
from .param_utils import params_prepare_for_save
from .utils import ChangeStatusRequest, validate_status_change
from .utils import ChangeStatusRequest, validate_status_change, update_project_time
log = config.logger(__file__)
org_bll = OrgBLL()
@@ -275,6 +275,7 @@ class TaskBLL:
tags=updated_tags,
system_tags=updated_system_tags,
)
update_project_time(new_task.project)
return new_task, new_project_data

View File

@@ -1,5 +1,5 @@
from datetime import datetime
from typing import TypeVar, Callable, Tuple, Sequence
from typing import TypeVar, Callable, Tuple, Sequence, Union
import attr
import six
@@ -153,9 +153,14 @@ def get_possible_status_changes(current_status):
return possible
def update_project_time(project_id):
if project_id:
Project.objects(id=project_id).update(last_update=datetime.utcnow())
def update_project_time(project_ids: Union[str, Sequence[str]]):
if not project_ids:
return
if isinstance(project_ids, str):
project_ids = [project_ids]
return Project.objects(id__in=project_ids).update(last_update=datetime.utcnow())
T = TypeVar("T")