ClearML/clearml
1
0
Fork 0
mirror of https://github.com/clearml/clearml synced 2025-02-12 07:35:08 +00:00
Code Issues Actions Packages Projects Releases Wiki Activity

Add sdk.development.worker.wait_for_first_iteration_to_start_sec and sdk.development.worker.max_wait_for_first_iteration_to_start_sec configuration options

Browse Source
This commit is contained in:
allegroai 2024-04-16 18:41:47 +03:00
parent c751bea4dc
commit 2717c7ea76
4 changed files with 28 additions and 3 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

10
clearml/config/default/sdk.conf
View File

@ -209,7 +209,15 @@
report_global_mem_used: false report_global_mem_used: false
# if provided, start resource reporting after this amount of seconds # if provided, start resource reporting after this amount of seconds
#report_start_sec: 30 # report_start_sec: 30
# set the initial time (seconds) to wait for iteration reporting to be used as x-axis for the
# resource monitoring, if timeout exceeds then reverts to "seconds from start"
# wait_for_first_iteration_to_start_sec: 30
# set the maximum time (seconds) to allow the resource monitoring to revert back to
# iteration reporting x-axis after starting to report "seconds from start"
# max_wait_for_first_iteration_to_start_sec: 1800
} }
} }

4
clearml/task.py
View File

@ -733,6 +733,10 @@ class Task(_Task):
task, task,
report_mem_used_per_process=not config.get('development.worker.report_global_mem_used', False), report_mem_used_per_process=not config.get('development.worker.report_global_mem_used', False),
first_report_sec=config.get('development.worker.report_start_sec', None), first_report_sec=config.get('development.worker.report_start_sec', None),
wait_for_first_iteration_to_start_sec=config.get(
'development.worker.wait_for_first_iteration_to_start_sec', None),
max_wait_for_first_iteration_to_start_sec=config.get(
'development.worker.max_wait_for_first_iteration_to_start_sec', None),
) )
task._resource_monitor.start() task._resource_monitor.start()

6
clearml/utilities/resource_monitor.py
View File

@ -31,8 +31,10 @@ class ResourceMonitor(BackgroundMonitor):
self._sample_frequency = sample_frequency_per_sec self._sample_frequency = sample_frequency_per_sec
self._report_frequency = report_frequency_sec self._report_frequency = report_frequency_sec
self._first_report_sec = first_report_sec or report_frequency_sec self._first_report_sec = first_report_sec or report_frequency_sec
self.wait_for_first_iteration = wait_for_first_iteration_to_start_sec self.wait_for_first_iteration = 180. if wait_for_first_iteration_to_start_sec is None \
self.max_check_first_iteration = max_wait_for_first_iteration_to_start_sec else wait_for_first_iteration_to_start_sec
self.max_check_first_iteration = 1800. if max_wait_for_first_iteration_to_start_sec is None \
else max_wait_for_first_iteration_to_start_sec
self._num_readouts = 0 self._num_readouts = 0
self._readouts = {} self._readouts = {}
self._previous_readouts = {} self._previous_readouts = {}

11
docs/clearml.conf
View File

@ -228,6 +228,17 @@ sdk {
# compatibility feature, report memory usage for the entire machine # compatibility feature, report memory usage for the entire machine
# default (false), report only on the running process and its sub-processes # default (false), report only on the running process and its sub-processes
report_global_mem_used: false report_global_mem_used: false
# if provided, start resource reporting after this amount of seconds
# report_start_sec: 30
# set the initial time (seconds) to wait for iteration reporting to be used as x-axis for the
# resource monitoring, if timeout exceeds then reverts to "seconds from start"
# wait_for_first_iteration_to_start_sec: 30
# set the maximum time (seconds) to allow the resource monitoring to revert back to
# iteration reporting x-axis after starting to report "seconds from start"
# max_wait_for_first_iteration_to_start_sec: 1800
} }
} }
} }