ClearML/clearml
1
0
Fork 0
mirror of https://github.com/clearml/clearml synced 2025-06-23 01:55:38 +00:00
Code Issues Actions Packages Projects Releases Wiki Activity

Add support for development.worker.report_start_sec to allow starting resource monitoring sooner than usual

Browse Source
This commit is contained in:
allegroai 2023-05-11 15:57:15 +03:00
parent 295280cb93
commit c2b6165987
2 changed files with 7 additions and 2 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

3
clearml/config/default/sdk.conf
View File

@ -205,6 +205,9 @@
# compatibility feature, report memory usage for the entire machine # compatibility feature, report memory usage for the entire machine
# default (false), report only on the running process and its sub-processes # default (false), report only on the running process and its sub-processes
report_global_mem_used: false report_global_mem_used: false
# if provided, start resource reporting after this amount of seconds
#report_start_sec: 30
} }
} }

6
clearml/task.py
View File

@ -702,8 +702,10 @@ class Task(_Task):
resource_monitor_cls = auto_resource_monitoring \ resource_monitor_cls = auto_resource_monitoring \
if isinstance(auto_resource_monitoring, six.class_types) else ResourceMonitor if isinstance(auto_resource_monitoring, six.class_types) else ResourceMonitor
task._resource_monitor = resource_monitor_cls( task._resource_monitor = resource_monitor_cls(
task, report_mem_used_per_process=not config.get( task,
'development.worker.report_global_mem_used', False)) report_mem_used_per_process=not config.get('development.worker.report_global_mem_used', False),
first_report_sec=config.get('development.worker.report_start_sec', None),
)
task._resource_monitor.start() task._resource_monitor.start()
# make sure all random generators are initialized with new seed # make sure all random generators are initialized with new seed