ClearML/clearml
1
0
Fork 0
mirror of https://github.com/clearml/clearml synced 2025-03-03 18:52:12 +00:00
Code Issues Actions Packages Projects Releases Wiki Activity

Fix HPO randomly aborts running tasks before the time limit

Browse Source
This commit is contained in:
allegroai 2022-03-24 19:38:30 +02:00
parent b1120195df
commit c6769b5fc0
1 changed files with 10 additions and 2 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

12
clearml/automation/optimization.py
View File

@ -1747,8 +1747,16 @@ class HyperParameterOptimizer(object):
self._report_remaining_budget(task_logger, counter)
if self.optimizer.budget.compute_time.used and self.optimizer.budget.compute_time.used >= 1.0:
# Reached compute time limit
if (
self.optimizer.budget.compute_time.used
and self.optimizer.budget.compute_time.limit
and self.optimizer.budget.compute_time.used >= self.optimizer.budget.compute_time.limit
):
logger.warning(
"Optimizer task reached compute time limit (used {:.2f} out of {:.2f})".format(
self.optimizer.budget.compute_time.limit, self.optimizer.compute_time.used
)
)
timeout = -1
self._report_resources(task_logger, counter)