From 327c9f60b66e8a1036ac5699f2d39819f2c9026c Mon Sep 17 00:00:00 2001 From: allegroai <> Date: Sun, 27 Oct 2019 00:32:23 +0300 Subject: [PATCH] Add simple AutoML examples --- .../automl_base_template_keras_simple.py | 86 +++++++++++++++++++ .../automl/automl_random_search_example.py | 45 ++++++++++ examples/automl/task_piping_example.py | 43 ++++++++++ examples/automl/toy_base_task.py | 19 ++++ 4 files changed, 193 insertions(+) create mode 100644 examples/automl/automl_base_template_keras_simple.py create mode 100644 examples/automl/automl_random_search_example.py create mode 100644 examples/automl/task_piping_example.py create mode 100644 examples/automl/toy_base_task.py diff --git a/examples/automl/automl_base_template_keras_simple.py b/examples/automl/automl_base_template_keras_simple.py new file mode 100644 index 00000000..f4faa14b --- /dev/null +++ b/examples/automl/automl_base_template_keras_simple.py @@ -0,0 +1,86 @@ +# TRAINS - Keras with Tensorboard example code, automatic logging model and Tensorboard outputs +# +# Train a simple deep NN on the MNIST dataset. +# Gets to 98.40% test accuracy after 20 epochs +# (there is *a lot* of margin for parameter tuning). +# 2 seconds per epoch on a K520 GPU. +from __future__ import print_function + +import tempfile +import os + +from keras.callbacks import TensorBoard, ModelCheckpoint +from keras.datasets import mnist +from keras.models import Sequential +from keras.layers.core import Dense, Activation +from keras.optimizers import RMSprop +from keras.utils import np_utils +import tensorflow as tf + +from trains import Task, Logger + + +# Connecting TRAINS +task = Task.init(project_name='examples', task_name='Keras AutoML base') + + +# the data, shuffled and split between train and test sets +nb_classes = 10 +(X_train, y_train), (X_test, y_test) = mnist.load_data() + +X_train = X_train.reshape(60000, 784).astype('float32')/255. +X_test = X_test.reshape(10000, 784).astype('float32')/255. +print(X_train.shape[0], 'train samples') +print(X_test.shape[0], 'test samples') + +# convert class vectors to binary class matrices +Y_train = np_utils.to_categorical(y_train, nb_classes) +Y_test = np_utils.to_categorical(y_test, nb_classes) + +args = {'batch_size': 128, + 'epochs': 6, + 'layer_1': 512, + 'layer_2': 512, + 'layer_3': 10, + 'layer_4': 512, + } +args = task.connect(args) + +model = Sequential() +model.add(Dense(args['layer_1'], input_shape=(784,))) +model.add(Activation('relu')) +# model.add(Dropout(0.2)) +model.add(Dense(args['layer_2'])) +model.add(Activation('relu')) +# model.add(Dropout(0.2)) +model.add(Dense(args['layer_3'])) +model.add(Activation('softmax')) + +model2 = Sequential() +model2.add(Dense(args['layer_4'], input_shape=(784,))) +model2.add(Activation('relu')) + +model.summary() + +model.compile(loss='categorical_crossentropy', + optimizer=RMSprop(), + metrics=['accuracy']) + +# Advanced: setting model class enumeration +labels = dict(('digit_%d' % i, i) for i in range(10)) +task.set_model_label_enumeration(labels) + +output_folder = os.path.join(tempfile.gettempdir(), 'keras_example') + +board = TensorBoard(log_dir=output_folder, write_images=False) +model_store = ModelCheckpoint(filepath=os.path.join(output_folder, 'weight.hdf5')) + +history = model.fit(X_train, Y_train, + batch_size=args['batch_size'], epochs=args['epochs'], + callbacks=[board, model_store], + validation_data=(X_test, Y_test)) +score = model.evaluate(X_test, Y_test, verbose=0) +print('Test score:', score[0]) +print('Test accuracy:', score[1]) +Logger.current_logger().report_scalar(title='evaluate', series='score', value=score[0], iteration=args['epochs']) +Logger.current_logger().report_scalar(title='evaluate', series='accuracy', value=score[1], iteration=args['epochs']) diff --git a/examples/automl/automl_random_search_example.py b/examples/automl/automl_random_search_example.py new file mode 100644 index 00000000..1b5db226 --- /dev/null +++ b/examples/automl/automl_random_search_example.py @@ -0,0 +1,45 @@ +from random import random, sample +from trains import Task + + +# define random search space, +# This is a simple random search +# (can be integrated with 'bayesian-optimization' 'hpbandster' etc.) +space = { + 'batch_size': lambda: sample([64, 96, 128, 160, 192], 1)[0], + 'layer_1': lambda: sample(range(128, 512, 32), 1)[0], + 'layer_2': lambda: sample(range(128, 512, 32), 1)[0], +} + +# number of random samples to test from 'space' +total_number_of_experiments = 3 + +# execution queue to add experiments to +execution_queue_name = 'default' + +# Select base template task +# Notice we can be more imaginative and use task_id which will eliminate the need to use project name +template_task = Task.get_task(project_name='examples', task_name='Keras AutoML base') + +for i in range(total_number_of_experiments): + # clone the template task into a new write enabled task (where we can change parameters) + cloned_task = Task.clone(source_task=template_task, + name=template_task.name+' {}'.format(i), parent=template_task.id) + + # get the original template parameters + cloned_task_parameters = cloned_task.get_parameters() + + # override with random samples form grid + for k in space.keys(): + cloned_task_parameters[k] = space[k]() + + # put back into the new cloned task + cloned_task.set_parameters(cloned_task_parameters) + print('Experiment {} set with parameters {}'.format(i, cloned_task_parameters)) + + # enqueue the task for execution + Task.enqueue(cloned_task.id, queue_name=execution_queue_name) + print('Experiment id={} enqueue for execution'.format(cloned_task.id)) + +# we are done, the next step is to watch the experiments graphs +print('Done') diff --git a/examples/automl/task_piping_example.py b/examples/automl/task_piping_example.py new file mode 100644 index 00000000..758e0adf --- /dev/null +++ b/examples/automl/task_piping_example.py @@ -0,0 +1,43 @@ +from trains import Task +from time import sleep + +# Initialize the Task Pipe's first Task used to start the Task Pipe +task = Task.init('examples', 'Simple Controller Task') + +# Create a hyper-parameter dictionary for the task +param = {} +# Connect the hyper-parameter dictionary to the task +param = task.connect(param) + +# In this example we pass next task's name as a parameter +param['next_task_name'] = 'Toy Base Task' +# This is a parameter name in the next task we want to change +param['param_name'] = 'Example_Param' +# This is the parameter value in the next task we want to change +param['param_name_new_value'] = 3 +# The queue where we want the template task (clone) to be sent to +param['execution_queue_name'] = 'default' + +# Simulate the work of a Task +print('Processing....') +sleep(2.0) +print('Done processing :)') + +# Get a reference to the task to pipe to. +next_task = Task.get_task(project_name=task.get_project_name(), task_name=param['next_task_name']) + +# Clone the task to pipe to. This creates a task with status Draft whose parameters can be modified. +cloned_task = Task.clone(source_task=next_task, name='Auto generated cloned task') + +# Get the original parameters of the Task, modify the value of one parameter, +# and set the parameters in the next Task +cloned_task_parameters = cloned_task.get_parameters() +cloned_task_parameters[param['param_name']] = param['param_name_new_value'] +cloned_task.set_parameters(cloned_task_parameters) + +# Enqueue the Task for execution. The enqueued Task must already exist in the trains platform +print('Enqueue next step in pipeline to queue: {}'.format(param['execution_queue_name'])) +Task.enqueue(cloned_task.id, queue_name=param['execution_queue_name']) + +# We are done. The next step in the pipe line is in charge of the pipeline now. +print('Done') diff --git a/examples/automl/toy_base_task.py b/examples/automl/toy_base_task.py new file mode 100644 index 00000000..b18765e4 --- /dev/null +++ b/examples/automl/toy_base_task.py @@ -0,0 +1,19 @@ +# This Task is the base task that we will be executing as a second step (see task_piping.py) +# In order to make sure this experiment is registered in the platform, you must execute it once. + +from trains import Task + +# Initialize the task pipe's first task used to start the task pipe +task = Task.init('examples', 'Toy Base Task') + +# Create a dictionary for hyper-parameters +params = {} + +# Add a parameter and value to the dictionary +params['Example_Param'] = 1 + +# Connect the hyper-parameter dictionary to the task +task.connect(params) + +# Print the value to demonstrate it is the value is set by the initiating task. +print ("Example_Param is", params['Example_Param'])