From 30c3968cd72f25857e432c45238d422ac30fab8e Mon Sep 17 00:00:00 2001 From: Victor Sonck Date: Tue, 28 Dec 2021 14:54:38 +0100 Subject: [PATCH] Add LocalClearmlJob as possible option for HPO (#525) --- clearml/automation/job.py | 3 +- clearml/automation/optimization.py | 28 ++++++++++++++++++- .../hyper_parameter_optimizer.py | 2 ++ 3 files changed, 31 insertions(+), 2 deletions(-) diff --git a/clearml/automation/job.py b/clearml/automation/job.py index 540bd6db..35eb261a 100644 --- a/clearml/automation/job.py +++ b/clearml/automation/job.py @@ -572,7 +572,8 @@ class ClearmlJob(BaseJob): class LocalClearmlJob(ClearmlJob): """ - Run jobs locally as a sub-process, use for debugging purposes only + Run jobs locally as a sub-process, use only when no agents are available (this will not use queues) + or for debug purposes. """ def __init__(self, *args, **kwargs): super(LocalClearmlJob, self).__init__(*args, **kwargs) diff --git a/clearml/automation/optimization.py b/clearml/automation/optimization.py index d98a7cb2..1885bbc3 100644 --- a/clearml/automation/optimization.py +++ b/clearml/automation/optimization.py @@ -9,7 +9,7 @@ from threading import Thread, Event from time import time from typing import List, Set, Union, Any, Sequence, Optional, Mapping, Callable -from .job import ClearmlJob +from .job import ClearmlJob, LocalClearmlJob from .parameters import Parameter from ..backend_interface.util import get_or_create_project from ..logger import Logger @@ -1268,6 +1268,32 @@ class HyperParameterOptimizer(object): return [] return [j.task for j in self.optimizer.get_running_jobs()] + def start_locally(self, job_complete_callback=None): + # type: (Optional[Callable[[str, float, int, dict, str], None]]) -> bool + """ + Start the HyperParameterOptimizer controller completely locally. Both the optimizer task + and all spawned substasks are ran on the local machine using the current environment. + If the calling process is stopped, then the controller stops as well. + + :param Callable job_complete_callback: Callback function, called when a job is completed. + + .. code-block:: py + + def job_complete_callback( + job_id, # type: str + objective_value, # type: float + objective_iteration, # type: int + job_parameters, # type: dict + top_performance_job_id # type: str + ): + pass + + :return: True, if the controller started. False, if the controller did not start. + + """ + self.optimizer.set_job_class(LocalClearmlJob) + return self.start(job_complete_callback=job_complete_callback) + def start(self, job_complete_callback=None): # type: (Optional[Callable[[str, float, int, dict, str], None]]) -> bool """ diff --git a/examples/optimization/hyper-parameter-optimization/hyper_parameter_optimizer.py b/examples/optimization/hyper-parameter-optimization/hyper_parameter_optimizer.py index 4af6e74d..0553108a 100644 --- a/examples/optimization/hyper-parameter-optimization/hyper_parameter_optimizer.py +++ b/examples/optimization/hyper-parameter-optimization/hyper_parameter_optimizer.py @@ -119,6 +119,8 @@ an_optimizer.set_report_period(2.2) # start the optimization process, callback function to be called every time an experiment is completed # this function returns immediately an_optimizer.start(job_complete_callback=job_complete_callback) +# You can also use the line below instead to run all the optimizer tasks locally, without using queues or agent +# an_optimizer.start_locally(job_complete_callback=job_complete_callback) # set the time limit for the optimization process (2 hours) an_optimizer.set_time_limit(in_minutes=120.0) # wait until process is done (notice we are controlling the optimization process in the background)