ClearML/clearml
1
0
Fork 0
mirror of https://github.com/clearml/clearml synced 2025-03-03 18:52:12 +00:00
Code Issues Actions Packages Projects Releases Wiki Activity

Fix GPU stats on Windows machines (issue #177)

Browse Source
This commit is contained in:
allegroai 2020-08-08 12:40:32 +03:00
parent 35fa4f010a
commit ff736a76c9
1 changed files with 30 additions and 23 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

53
trains/utilities/gpu/gpustat.py
View File

@ -200,24 +200,30 @@ class GPUStatCollection(object):
GPUStatCollection.global_processes[nv_process.pid] = \ GPUStatCollection.global_processes[nv_process.pid] = \
psutil.Process(pid=nv_process.pid) psutil.Process(pid=nv_process.pid)
ps_process = GPUStatCollection.global_processes[nv_process.pid] ps_process = GPUStatCollection.global_processes[nv_process.pid]
process['username'] = ps_process.username()
# cmdline returns full path;
# as in `ps -o comm`, get short cmdnames.
_cmdline = ps_process.cmdline()
if not _cmdline:
# sometimes, zombie or unknown (e.g. [kworker/8:2H])
process['command'] = '?'
process['full_command'] = ['?']
else:
process['command'] = os.path.basename(_cmdline[0])
process['full_command'] = _cmdline
# Bytes to MBytes
process['gpu_memory_usage'] = nv_process.usedGpuMemory // MB
process['cpu_percent'] = ps_process.cpu_percent()
process['cpu_memory_usage'] = \
round((ps_process.memory_percent() / 100.0) *
psutil.virtual_memory().total)
process['pid'] = nv_process.pid process['pid'] = nv_process.pid
# noinspection PyBroadException
try:
# we do not actually use these, so no point in collecting them
# process['username'] = ps_process.username()
# # cmdline returns full path;
# # as in `ps -o comm`, get short cmdnames.
# _cmdline = ps_process.cmdline()
# if not _cmdline:
# # sometimes, zombie or unknown (e.g. [kworker/8:2H])
# process['command'] = '?'
# process['full_command'] = ['?']
# else:
# process['command'] = os.path.basename(_cmdline[0])
# process['full_command'] = _cmdline
# process['cpu_percent'] = ps_process.cpu_percent()
# process['cpu_memory_usage'] = \
# round((ps_process.memory_percent() / 100.0) *
# psutil.virtual_memory().total)
# Bytes to MBytes
process['gpu_memory_usage'] = nv_process.usedGpuMemory // MB
except Exception:
# insufficient permissions
pass
return process return process
if not GPUStatCollection._gpu_device_info.get(index): if not GPUStatCollection._gpu_device_info.get(index):
@ -285,12 +291,13 @@ class GPUStatCollection(object):
# e.g. nvidia-smi reset or reboot the system # e.g. nvidia-smi reset or reboot the system
pass pass
# TODO: Do not block if full process info is not requested # we do not actually use these, so no point in collecting them
time.sleep(0.1) # # TODO: Do not block if full process info is not requested
for process in processes: # time.sleep(0.1)
pid = process['pid'] # for process in processes:
cache_process = GPUStatCollection.global_processes[pid] # pid = process['pid']
process['cpu_percent'] = cache_process.cpu_percent() # cache_process = GPUStatCollection.global_processes[pid]
# process['cpu_percent'] = cache_process.cpu_percent()
index = N.nvmlDeviceGetIndex(handle) index = N.nvmlDeviceGetIndex(handle)
gpu_info = { gpu_info = {