clearml/examples/frameworks/pytorch/notebooks/image/hyperparameter_search.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# execute this in command line on all machines to be used as workers before initiating the hyperparamer search \n",
    "# ! pip install -U trains-agent==0.15.0\n",
    "# ! trains-agent daemon --queue default\n",
    "\n",
    "# pip install with locked versions\n",
    "! pip install -U pandas==1.0.3\n",
    "! pip install -U trains>=0.16.2\n",
    "! pip install -U optuna==2.0.0"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "from trains.automation import UniformParameterRange, UniformIntegerParameterRange\n",
    "from trains.automation import HyperParameterOptimizer\n",
    "from trains.automation.optuna import OptimizerOptuna\n",
    "\n",
    "from trains import Task"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "task = Task.init(project_name='Hyperparameter Optimization with Optuna',\n",
    "                 task_name='Hyperparameter Search',\n",
    "                 task_type=Task.TaskTypes.optimizer)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#####################################################################\n",
    "### Don't forget to replace this default id with your own task id ###\n",
    "#####################################################################\n",
    "TEMPLATE_TASK_ID = 'b634a59993f8477f9e22167bae662be4'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "optimizer = HyperParameterOptimizer(\n",
    "    base_task_id=TEMPLATE_TASK_ID,  # This is the experiment we want to optimize\n",
    "    # here we define the hyper-parameters to optimize\n",
    "    hyper_parameters=[\n",
    "        UniformIntegerParameterRange('number_of_epochs', min_value=2, max_value=12, step_size=2),\n",
    "        UniformIntegerParameterRange('batch_size', min_value=2, max_value=16, step_size=2),\n",
    "        UniformParameterRange('dropout', min_value=0, max_value=0.5, step_size=0.05),\n",
    "        UniformParameterRange('base_lr', min_value=0.00025, max_value=0.01, step_size=0.00025),\n",
    "    ],\n",
    "    # setting the objective metric we want to maximize/minimize\n",
    "    objective_metric_title='accuracy',\n",
    "    objective_metric_series='total',\n",
    "    objective_metric_sign='max',  # maximize or minimize the objective metric\n",
    "\n",
    "    # setting optimizer - trains supports GridSearch, RandomSearch, OptimizerBOHB and OptimizerOptuna\n",
    "    optimizer_class=OptimizerOptuna,\n",
    "    \n",
    "    # Configuring optimization parameters\n",
    "    execution_queue='dan_queue',  # queue to schedule the experiments for execution\n",
    "    max_number_of_concurrent_tasks=2,  # number of concurrent experiments\n",
    "    optimization_time_limit=60.,  # set the time limit for the optimization process\n",
    "    compute_time_limit=120,  # set the compute time limit (sum of execution time on all machines)\n",
    "    total_max_jobs=20,  # set the maximum number of experiments for the optimization. \n",
    "                        # Converted to total number of iteration for OptimizerBOHB\n",
    "    min_iteration_per_job=15000,  # minimum number of iterations per experiment, till early stopping\n",
    "    max_iteration_per_job=150000,  # maximum number of iterations per experiment\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "optimizer.set_report_period(1)  # setting the time gap between two consecutive reports\n",
    "optimizer.start()  \n",
    "optimizer.wait()  # wait until process is done\n",
    "optimizer.stop()  # make sure background optimization stopped"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# optimization is completed, print the top performing experiments id\n",
    "k = 3\n",
    "top_exp = optimizer.get_top_experiments(top_k=k)\n",
    "print('Top {} experiments are:'.format(k))\n",
    "for n, t in enumerate(top_exp, 1):\n",
    "    print('Rank {}: task id={} |result={}'\n",
    "          .format(n, t.id, t.get_last_scalar_metrics()['accuracy']['total']['last']))"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.4"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}