ClearML/clearml
1
0
Fork 0
mirror of https://github.com/clearml/clearml synced 2025-05-24 05:54:04 +00:00
Code Issues Actions Packages Projects Releases Wiki Activity

Fix GPU reporting when NVIDIA_VISIBLE_DEVICES contains a directory reference

Browse Source
This commit is contained in:
clearml 2025-05-21 10:06:26 +03:00
parent dc4241eb0c
commit 2486dd7b9c
1 changed files with 9 additions and 1 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

10
clearml/utilities/resource_monitor.py
View File

@ -108,7 +108,15 @@ class ResourceMonitor(BackgroundMonitor):
try:
active_gpus = os.environ.get("NVIDIA_VISIBLE_DEVICES", "") or os.environ.get("CUDA_VISIBLE_DEVICES", "")
if active_gpus and active_gpus != "all":
self._active_gpus = [g.strip() for g in active_gpus.split(",")]
if os.path.isdir(active_gpus):
try:
self._active_gpus = os.listdir(active_gpus)
except OSError as e:
logging.getLogger("clearml.resource_monitor").warning(
"Failed listing {}: {}".format(active_gpus, e)
)
else:
self._active_gpus = [g.strip() for g in active_gpus.split(",")]
except Exception:
pass