ClearML/clearml-agent
1
0
Fork 0
mirror of https://github.com/clearml/clearml-agent synced 2025-03-03 18:52:22 +00:00
Code Issues Actions Packages Projects Releases Wiki Activity

Fix dynamic gpus priority queue

Browse Source
This commit is contained in:
allegroai 2021-04-20 18:11:59 +03:00
parent a671692832
commit 6f8d5710d6
1 changed files with 4 additions and 3 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

7
clearml_agent/commands/worker.py
View File

@ -774,9 +774,10 @@ class Worker(ServiceCommandSection):
if gpu_queues[queue][0] > len(available_gpus):
# not enough available_gpus, we should sleep and start over
if self._daemon_foreground or worker_params.debug:
print("Not enough free GPUs for queue={} {}/{}".format(
queue, len(available_gpus), gpu_queues[queue][0]))
continue
print("Not enough free GPUs {}/{}, sleeping for {:.1f} seconds".format(
len(available_gpus), gpu_queues[queue][0], self._polling_interval))
sleep(self._polling_interval)
break
# get next task in queue
try: