ClearML/clearml-agent
1
0
Fork 0
mirror of https://github.com/clearml/clearml-agent synced 2025-02-07 13:26:08 +00:00
Code Issues Actions Packages Projects Releases Wiki Activity

Fix pinging running task (and change default to once a minute)

Browse Source
This commit is contained in:
allegroai 2022-12-13 15:26:26 +02:00
parent f69e16ea9d
commit da19ef26c4
1 changed files with 6 additions and 3 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

9
clearml_agent/commands/worker.py
View File

@ -686,7 +686,7 @@ class Worker(ServiceCommandSection):
self._docker_args_filters = [] self._docker_args_filters = []
self._task_ping_interval_sec = max( self._task_ping_interval_sec = max(
0, text_to_int(self._session.config.get("agent.task_ping_interval_sec", 120.0)) 0, text_to_int(self._session.config.get("agent.task_ping_interval_sec", 60.0))
) )
@classmethod @classmethod
@ -1784,7 +1784,8 @@ class Worker(ServiceCommandSection):
if stderr: if stderr:
stderr.flush() stderr.flush()
if self._task_ping_interval_sec and time() - last_task_ping > self._task_ping_interval_sec: if not stopping and self._task_ping_interval_sec and \
time() - last_task_ping > self._task_ping_interval_sec:
# noinspection PyBroadException # noinspection PyBroadException
try: try:
res = (session or self._session).send(tasks_api.PingRequest(task=task_id)) res = (session or self._session).send(tasks_api.PingRequest(task=task_id))
@ -1793,7 +1794,7 @@ class Worker(ServiceCommandSection):
except Exception as ex: except Exception as ex:
self.log.error("Failed sending ping: %s", str(ex)) self.log.error("Failed sending ping: %s", str(ex))
finally: finally:
self._task_ping_interval_sec = time() last_task_ping = time()
# get diff from previous poll # get diff from previous poll
printed_lines, stdout_pos_count = _print_file(stdout_path, stdout_pos_count) printed_lines, stdout_pos_count = _print_file(stdout_path, stdout_pos_count)
@ -2162,6 +2163,8 @@ class Worker(ServiceCommandSection):
print(commit_docker(container_name=target, docker_id=docker_id, apply_change=change)) print(commit_docker(container_name=target, docker_id=docker_id, apply_change=change))
shutdown_docker_process(docker_id=docker_id) shutdown_docker_process(docker_id=docker_id)
safe_remove_file(temp_config.as_posix())
return return
def _get_task_python_version(self, task): def _get_task_python_version(self, task):