ClearML/clearml-agent
1
0
Fork 0
mirror of https://github.com/clearml/clearml-agent synced 2025-01-31 00:56:53 +00:00
Code Issues Actions Packages Projects Releases Wiki Activity

Fix only set Task started status on node rank 0

Browse Source
This commit is contained in:
allegroai 2024-08-27 22:52:31 +03:00
parent b21665ed6e
commit a4d3b5bad6
1 changed files with 15 additions and 12 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

27
clearml_agent/commands/worker.py
View File

@ -940,12 +940,14 @@ class Worker(ServiceCommandSection):
print("Warning: failed obtaining/setting hostname for task '{}': {}".format(task_id, ex))
# set task status to in_progress so we know it was popped from the queue
# noinspection PyBroadException
try:
task_session.send_api(tasks_api.StartedRequest(task=task_id, status_message="launch by agent", force=True))
except Exception:
print("Warning: Could not set status=in_progress task id '{}', skipping".format(task_id))
return
if not self._get_node_rank():
# noinspection PyBroadException
try:
task_session.send_api(tasks_api.StartedRequest(task=task_id, status_message="launch by agent", force=True))
except Exception:
print("Warning: Could not set status=in_progress task id '{}', skipping".format(task_id))
return
# setup console log
temp_stdout_name = safe_mkstemp(
suffix=".txt", prefix=".clearml_agent_out.", name_only=True, dir=(ENV_TEMP_STDOUT_FILE_DIR.get() or None)
@ -1336,12 +1338,13 @@ class Worker(ServiceCommandSection):
# set task status to in_progress so we know it was popped from the queue
# next api version we will set the status when pulling from the queue
# noinspection PyBroadException
try:
self._session.send_api(
tasks_api.StartedRequest(task=task_id, status_message="pulled by agent", force=True))
except Exception:
print("Warning: Could not set status=in_progress task id '{}', retrying in a bit".format(task_id))
if not self._get_node_rank():
# noinspection PyBroadException
try:
self._session.send_api(
tasks_api.StartedRequest(task=task_id, status_message="pulled by agent", force=True))
except Exception:
print("Warning: Could not set status=in_progress task id '{}', retrying in a bit".format(task_id))
# check if we need to impersonate
task_session = None