clearml/examples/frameworks/pytorch/notebooks/image/hyperparameter_search.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# execute this in command line on all machines to be used as workers before initiating the hyperparamer search \n",
    "# ! pip install -U trains-agent==0.15.0\n",
    "# ! trains-agent daemon --queue default\n",
    "\n",
    "# pip install with locked versions\n",
    "! pip install -U pandas==1.0.3\n",
    "! pip install -U trains>=0.15.0\n",
    "! pip install -U optuna==2.0.0rc0"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "from trains.automation import UniformParameterRange, UniformIntegerParameterRange\n",
    "from trains.automation import HyperParameterOptimizer\n",
    "from trains.automation.optuna import OptimizerOptuna\n",
    "\n",
    "from trains import Task"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "task = Task.init(project_name='Hyper-Parameter Search', task_name='Hyper-Parameter Optimization')\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#####################################################################\n",
    "### Don't forget to replace this default id with your own task id ###\n",
    "#####################################################################\n",
    "TEMPLATE_TASK_ID = 'd551a9990cb5451c9c744cc58201c612'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "optimizer = HyperParameterOptimizer(\n",
    "    base_task_id=TEMPLATE_TASK_ID,  # This is the experiment we want to optimize\n",
    "    # here we define the hyper-parameters to optimize\n",
    "    hyper_parameters=[\n",
    "        UniformIntegerParameterRange('number_of_epochs', min_value=5, max_value=15, step_size=1),\n",
    "        UniformIntegerParameterRange('batch_size', min_value=2, max_value=12, step_size=2),\n",
    "        UniformParameterRange('dropout', min_value=0, max_value=0.5, step_size=0.05),\n",
    "        UniformParameterRange('base_lr', min_value=0.0005, max_value=0.01, step_size=0.0005),\n",
    "    ],\n",
    "    # this is the objective metric we want to maximize/minimize\n",
    "    objective_metric_title='accuracy',\n",
    "    objective_metric_series='total',\n",
    "    objective_metric_sign='max',  # maximize or minimize the objective metric\n",
    "    max_number_of_concurrent_tasks=3,  # number of concurrent experiments\n",
    "    # setting optimizer - trains supports GridSearch, RandomSearch or OptimizerBOHB\n",
    "    optimizer_class=OptimizerOptuna,  # can be replaced with OptimizerBOHB\n",
    "    execution_queue='default',  # queue to schedule the experiments for execution\n",
    "    optimization_time_limit=30.,  # time limit for each experiment (optional, ignored by OptimizerBOHB)\n",
    "    pool_period_min=1,  # Check the experiments every x minutes\n",
    "    # set the maximum number of experiments for the optimization.\n",
    "    # OptimizerBOHB sets the total number of iteration as total_max_jobs * max_iteration_per_job\n",
    "    total_max_jobs=12,\n",
    "    # setting OptimizerBOHB configuration (ignored by other optimizers)\n",
    "    min_iteration_per_job=15000,  # minimum number of iterations per experiment, till early stopping\n",
    "    max_iteration_per_job=150000,  # maximum number of iterations per experiment\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "optimizer.set_time_limit(in_minutes=90.0)  # set the time limit for the optimization process\n",
    "optimizer.start()  \n",
    "optimizer.wait()  # wait until process is done\n",
    "optimizer.stop()  # make sure background optimization stopped"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# optimization is completed, print the top performing experiments id\n",
    "k = 3\n",
    "top_exp = optimizer.get_top_experiments(top_k=k)\n",
    "print('Top {} experiments are:'.format(k))\n",
    "for n, t in enumerate(top_exp, 1):\n",
    "    print('Rank {}: task id={} |result={}'\n",
    "          .format(n, t.id, t.get_last_scalar_metrics()['accuracy']['total']['last']))"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.4"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}