From a4b507598b88403be6b8cce4abed1c709cb64eda Mon Sep 17 00:00:00 2001 From: allegroai <> Date: Fri, 17 Dec 2021 11:55:46 +0200 Subject: [PATCH] Fix resource monitor fails on permission issues (skip over parts) --- clearml/utilities/resource_monitor.py | 23 +++++++++++++++++------ 1 file changed, 17 insertions(+), 6 deletions(-) diff --git a/clearml/utilities/resource_monitor.py b/clearml/utilities/resource_monitor.py index 8a6ceb26..88eaafed 100644 --- a/clearml/utilities/resource_monitor.py +++ b/clearml/utilities/resource_monitor.py @@ -214,13 +214,24 @@ class ResourceMonitor(BackgroundMonitor): if "coretemp" in sensor_stat and len(sensor_stat["coretemp"]): stats["cpu_temperature"] = max([float(t.current) for t in sensor_stat["coretemp"]]) + # protect against permission issues # update cached measurements - net_stats = psutil.net_io_counters() - stats["network_tx_mbs"] = bytes_to_megabytes(net_stats.bytes_sent) - stats["network_rx_mbs"] = bytes_to_megabytes(net_stats.bytes_recv) - io_stats = psutil.disk_io_counters() - stats["io_read_mbs"] = bytes_to_megabytes(io_stats.read_bytes) - stats["io_write_mbs"] = bytes_to_megabytes(io_stats.write_bytes) + # noinspection PyBroadException + try: + net_stats = psutil.net_io_counters() + stats["network_tx_mbs"] = bytes_to_megabytes(net_stats.bytes_sent) + stats["network_rx_mbs"] = bytes_to_megabytes(net_stats.bytes_recv) + except Exception: + pass + + # protect against permission issues + # noinspection PyBroadException + try: + io_stats = psutil.disk_io_counters() + stats["io_read_mbs"] = bytes_to_megabytes(io_stats.read_bytes) + stats["io_write_mbs"] = bytes_to_megabytes(io_stats.write_bytes) + except Exception: + pass # check if we can access the gpu statistics if self._gpustat: