ClearML/clearml-agent
1
0
Fork 0
mirror of https://github.com/clearml/clearml-agent synced 2025-06-26 18:16:15 +00:00
Code Issues Actions Packages Projects Releases Wiki Activity

Protect dynamic GPUs from failing to parse worker GPU index

Browse Source
This commit is contained in:
allegroai 2022-04-13 10:01:50 +03:00
parent 1011544533
commit f50ba005b5
1 changed files with 5 additions and 1 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

6
clearml_agent/commands/worker.py
View File

@ -1109,7 +1109,11 @@ class Worker(ServiceCommandSection):
if w.id.startswith(worker_name) and w.id != self.worker_id] if w.id.startswith(worker_name) and w.id != self.worker_id]
gpus = [] gpus = []
for w in our_workers: for w in our_workers:
gpus += [int(g) for g in w.split(':')[-1].lower().replace('gpu', '').split(',')] for g in w.split(':')[-1].lower().replace('gpu', '').split(','):
try:
gpus += [int(g.strip())]
except (ValueError, TypeError):
print("INFO: failed parsing GPU int('{}') - skipping".format(g))
available_gpus = list(set(gpu_indexes) - set(gpus)) available_gpus = list(set(gpu_indexes) - set(gpus))
return available_gpus return available_gpus