ClearML/clearml-agent
1
0
Fork 0
mirror of https://github.com/clearml/clearml-agent synced 2025-02-12 07:38:04 +00:00
Code Issues Actions Packages Projects Releases Wiki Activity

Fix k8s glue, do not reset Task before re-enqueuing as it will remove runtime properties

Browse Source
This commit is contained in:
allegroai 2021-07-15 10:27:54 +03:00
parent ca360b7d43
commit 499b3dfa66
1 changed files with 8 additions and 5 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

13
clearml_agent/glue/k8s.py
View File

@ -293,9 +293,12 @@ class K8sIntegration(Worker):
# push task into the k8s queue, so we have visibility on pending tasks in the k8s scheduler # push task into the k8s queue, so we have visibility on pending tasks in the k8s scheduler
try: try:
print('Pushing task {} into temporary pending queue'.format(task_id)) print('Pushing task {} into temporary pending queue'.format(task_id))
self._session.api_client.tasks.reset(task_id) self._session.api_client.tasks.stop(task_id, force=True)
self._session.api_client.tasks.enqueue(task_id, queue=self.k8s_pending_queue_name, self._session.api_client.tasks.enqueue(
status_reason='k8s pending scheduler') task_id,
queue=self.k8s_pending_queue_name,
status_reason='k8s pending scheduler'
)
except Exception as e: except Exception as e:
self.log.error("ERROR: Could not push back task [{}] to k8s pending queue [{}], error: {}".format( self.log.error("ERROR: Could not push back task [{}] to k8s pending queue [{}], error: {}".format(
task_id, self.k8s_pending_queue_name, e)) task_id, self.k8s_pending_queue_name, e))
@ -367,7 +370,7 @@ class K8sIntegration(Worker):
output, task_id, queue, ex output, task_id, queue, ex
) )
) )
self._session.api_client.tasks.reset(task_id) self._session.api_client.tasks.stop(task_id, force=True)
self._session.api_client.tasks.enqueue(task_id, queue=queue, status_reason='kubectl parsing error') self._session.api_client.tasks.enqueue(task_id, queue=queue, status_reason='kubectl parsing error')
return return
max_count = self.max_pods_limit max_count = self.max_pods_limit
@ -385,7 +388,7 @@ class K8sIntegration(Worker):
task_id, queue task_id, queue
) )
) )
self._session.api_client.tasks.reset(task_id) self._session.api_client.tasks.stop(task_id, force=True)
self._session.api_client.tasks.enqueue( self._session.api_client.tasks.enqueue(
task_id, queue=queue, status_reason='k8s max pod limit (no free k8s service)') task_id, queue=queue, status_reason='k8s max pod limit (no free k8s service)')
return return