ClearML/clearml
1
0
Fork 0
mirror of https://github.com/clearml/clearml synced 2025-02-07 21:33:25 +00:00
Code Issues Actions Packages Projects Releases Wiki Activity

Merge pull request #170 from GuillaumeTh/BF_GPU_monitoring

Browse Source 
Robustify GPU monitoring
This commit is contained in:
Allegro AI 2020-07-15 18:23:08 +03:00 committed by GitHub
commit 17f7d51a93
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
1 changed files with 3 additions and 0 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

3
trains/utilities/resource_monitor.py
View File

@ -299,6 +299,9 @@ class ResourceMonitor(object):
gpu_stat = self._gpustat.new_query(per_process_stats=True)
gpu_mem = {}
for i, g in enumerate(gpu_stat.gpus):
# only monitor the active gpu's, if none were selected, monitor everything
if self._active_gpus and i not in self._active_gpus:
continue
gpu_mem[i] = 0
for p in g.processes:
if p['pid'] in self._last_process_id_list: