Add LocalClearmlJob as possible option for HPO (#525)

clearml/automation/job.py

@@ -572,7 +572,8 @@ class ClearmlJob(BaseJob):
 
 class LocalClearmlJob(ClearmlJob):
     """
-    Run jobs locally as a sub-process, use for debugging purposes only
+    Run jobs locally as a sub-process, use only when no agents are available (this will not use queues)
+    or for debug purposes.
     """
     def __init__(self, *args, **kwargs):
         super(LocalClearmlJob, self).__init__(*args, **kwargs)

clearml/automation/optimization.py

@@ -9,7 +9,7 @@ from threading import Thread, Event
 from time import time
 from typing import List, Set, Union, Any, Sequence, Optional, Mapping, Callable
 
-from .job import ClearmlJob
+from .job import ClearmlJob, LocalClearmlJob
 from .parameters import Parameter
 from ..backend_interface.util import get_or_create_project
 from ..logger import Logger
@@ -1268,6 +1268,32 @@ class HyperParameterOptimizer(object):
             return []
         return [j.task for j in self.optimizer.get_running_jobs()]
 
+    def start_locally(self, job_complete_callback=None):
+        # type: (Optional[Callable[[str, float, int, dict, str], None]]) -> bool
+        """
+        Start the HyperParameterOptimizer controller completely locally. Both the optimizer task
+        and all spawned substasks are ran on the local machine using the current environment.
+        If the calling process is stopped, then the controller stops as well.
+
+        :param Callable job_complete_callback: Callback function, called when a job is completed.
+
+            .. code-block:: py
+
+                def job_complete_callback(
+                    job_id,                 # type: str
+                    objective_value,        # type: float
+                    objective_iteration,    # type: int
+                    job_parameters,         # type: dict
+                    top_performance_job_id  # type: str
+                ):
+                    pass
+
+        :return: True, if the controller started. False, if the controller did not start.
+
+        """
+        self.optimizer.set_job_class(LocalClearmlJob)
+        return self.start(job_complete_callback=job_complete_callback)
+
     def start(self, job_complete_callback=None):
         # type: (Optional[Callable[[str, float, int, dict, str], None]]) -> bool
         """

examples/optimization/hyper-parameter-optimization/hyper_parameter_optimizer.py

@@ -119,6 +119,8 @@ an_optimizer.set_report_period(2.2)
 # start the optimization process, callback function to be called every time an experiment is completed
 # this function returns immediately
 an_optimizer.start(job_complete_callback=job_complete_callback)
+# You can also use the line below instead to run all the optimizer tasks locally, without using queues or agent
+# an_optimizer.start_locally(job_complete_callback=job_complete_callback)
 # set the time limit for the optimization process (2 hours)
 an_optimizer.set_time_limit(in_minutes=120.0)
 # wait until process is done (notice we are controlling the optimization process in the background)