ClearML/clearml-agent
1
0
Fork 0
mirror of https://github.com/clearml/clearml-agent synced 2025-02-07 13:26:08 +00:00
Code Issues Actions Packages Projects Releases Wiki Activity

Fix dynamic GPUs with "all" GPUs on he same worker

Browse Source
This commit is contained in:
allegroai 2022-04-20 08:56:22 +03:00
parent 6e31171d31
commit 2466eed23f
1 changed files with 7 additions and 1 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

8
clearml_agent/commands/worker.py
View File

@ -1124,7 +1124,13 @@ class Worker(ServiceCommandSection):
raise ValueError("Dynamic GPU allocation is not supported by the ClearML-server")
available_gpus = [prop["value"] for prop in available_gpus if prop["key"] == 'available_gpus']
if available_gpus:
available_gpus = [int(g) for g in available_gpus[-1].split(',')]
gpus = []
for g in available_gpus[-1].split(','):
try:
gpus += [int(g.strip())]
except (ValueError, TypeError):
print("INFO: failed parsing GPU int('{}') - skipping".format(g))
available_gpus = gpus
if not isinstance(gpu_queues, dict):
gpu_queues = dict(gpu_queues)