ClearML/clearml
1
0
Fork 0
mirror of https://github.com/clearml/clearml synced 2025-03-03 10:42:00 +00:00
Code Issues Actions Packages Projects Releases Wiki Activity

Robustify GPU monitoring

Browse Source
This commit is contained in:
Guillaume Theaud 2020-07-15 11:16:05 -04:00
parent e7864e6ba8
commit 15afc479c6
1 changed files with 3 additions and 0 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

3
trains/utilities/resource_monitor.py
View File

@ -299,6 +299,9 @@ class ResourceMonitor(object):
gpu_stat = self._gpustat.new_query(per_process_stats=True)
gpu_mem = {}
for i, g in enumerate(gpu_stat.gpus):
# only monitor the active gpu's, if none were selected, monitor everything
if self._active_gpus and i not in self._active_gpus:
continue
gpu_mem[i] = 0
for p in g.processes:
if p['pid'] in self._last_process_id_list: