ClearML/clearml
1
0
Fork 0
mirror of https://github.com/clearml/clearml synced 2025-02-07 13:23:40 +00:00
Code Issues Actions Packages Projects Releases Wiki Activity

Don't send GPU specs if not available

Browse Source
This commit is contained in:
allegroai 2021-08-05 18:16:29 +03:00
parent d585535dcd
commit c7e5abd12e
1 changed files with 7 additions and 9 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

16
clearml/utilities/resource_monitor.py
View File

@ -348,20 +348,18 @@ class ResourceMonitor(BackgroundMonitor):
'memory_gb': round(psutil.virtual_memory().total / 1024 ** 3, 1),
'hostname': str(platform.node()),
'gpu_count': 0,
'gpu_type': 'N/A',
'gpu_memory': 'N/A',
'gpu_driver_version': 'N/A',
'gpu_driver_cuda_version': 'N/A',
}
if self._gpustat:
gpu_stat = self._gpustat.new_query(shutdown=True, get_driver_info=True)
if gpu_stat.gpus:
gpus = [g for i, g in enumerate(gpu_stat.gpus) if not self._active_gpus or i in self._active_gpus]
specs['gpu_count'] = int(len(gpus))
specs['gpu_type'] = ', '.join(g.name for g in gpus)
specs['gpu_memory'] = ', '.join('{}GB'.format(round(g.memory_total/1024.0)) for g in gpus)
specs['gpu_driver_version'] = gpu_stat.driver_version or ''
specs['gpu_driver_cuda_version'] = gpu_stat.driver_cuda_version or ''
specs.update(
gpu_count=int(len(gpus)),
gpu_type=', '.join(g.name for g in gpus),
gpu_memory=', '.join('{}GB'.format(round(g.memory_total/1024.0)) for g in gpus),
gpu_driver_version=gpu_stat.driver_version or '',
gpu_driver_cuda_version=gpu_stat.driver_cuda_version or '',
)
except Exception:
pass