ClearML/clearml
1
0
Fork 0
mirror of https://github.com/clearml/clearml synced 2025-03-03 18:52:12 +00:00
Code Issues Actions Packages Projects Releases Wiki Activity

Fix resource monitor fails on permission issues (skip over parts)

Browse Source
This commit is contained in:
allegroai 2021-12-17 11:55:46 +02:00
parent 24464b7c10
commit a4b507598b
1 changed files with 17 additions and 6 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

23
clearml/utilities/resource_monitor.py
View File

@ -214,13 +214,24 @@ class ResourceMonitor(BackgroundMonitor):
if "coretemp" in sensor_stat and len(sensor_stat["coretemp"]): if "coretemp" in sensor_stat and len(sensor_stat["coretemp"]):
stats["cpu_temperature"] = max([float(t.current) for t in sensor_stat["coretemp"]]) stats["cpu_temperature"] = max([float(t.current) for t in sensor_stat["coretemp"]])
# protect against permission issues
# update cached measurements # update cached measurements
net_stats = psutil.net_io_counters() # noinspection PyBroadException
stats["network_tx_mbs"] = bytes_to_megabytes(net_stats.bytes_sent) try:
stats["network_rx_mbs"] = bytes_to_megabytes(net_stats.bytes_recv) net_stats = psutil.net_io_counters()
io_stats = psutil.disk_io_counters() stats["network_tx_mbs"] = bytes_to_megabytes(net_stats.bytes_sent)
stats["io_read_mbs"] = bytes_to_megabytes(io_stats.read_bytes) stats["network_rx_mbs"] = bytes_to_megabytes(net_stats.bytes_recv)
stats["io_write_mbs"] = bytes_to_megabytes(io_stats.write_bytes) except Exception:
pass
# protect against permission issues
# noinspection PyBroadException
try:
io_stats = psutil.disk_io_counters()
stats["io_read_mbs"] = bytes_to_megabytes(io_stats.read_bytes)
stats["io_write_mbs"] = bytes_to_megabytes(io_stats.write_bytes)
except Exception:
pass
# check if we can access the gpu statistics # check if we can access the gpu statistics
if self._gpustat: if self._gpustat: