mirror of
https://github.com/clearml/clearml
synced 2025-06-26 18:16:07 +00:00
Documentation examples
This commit is contained in:
parent
7edc998824
commit
a29a655a6e
@ -2,7 +2,8 @@ from random import sample
|
|||||||
|
|
||||||
from clearml import Task
|
from clearml import Task
|
||||||
|
|
||||||
# Connecting ClearML
|
# Connecting ClearML with the current process,
|
||||||
|
# from here on everything is logged automatically
|
||||||
task = Task.init(project_name='examples', task_name='Random Hyper-Parameter Search Example', task_type=Task.TaskTypes.optimizer)
|
task = Task.init(project_name='examples', task_name='Random Hyper-Parameter Search Example', task_type=Task.TaskTypes.optimizer)
|
||||||
|
|
||||||
# Create a hyper-parameter dictionary for the task
|
# Create a hyper-parameter dictionary for the task
|
||||||
|
@ -5,6 +5,8 @@ from tensorflow import keras
|
|||||||
|
|
||||||
from clearml import Task
|
from clearml import Task
|
||||||
|
|
||||||
|
# Connecting ClearML with the current process,
|
||||||
|
# from here on everything is logged automatically
|
||||||
task = Task.init(project_name="autokeras", task_name="autokeras imdb example with scalars")
|
task = Task.init(project_name="autokeras", task_name="autokeras imdb example with scalars")
|
||||||
|
|
||||||
|
|
||||||
|
@ -6,6 +6,8 @@ from fastai.vision import * # Quick access to computer vision functionality
|
|||||||
|
|
||||||
from clearml import Task
|
from clearml import Task
|
||||||
|
|
||||||
|
# Connecting ClearML with the current process,
|
||||||
|
# from here on everything is logged automatically
|
||||||
task = Task.init(project_name="example", task_name="fastai with tensorboard callback")
|
task = Task.init(project_name="example", task_name="fastai with tensorboard callback")
|
||||||
|
|
||||||
path = untar_data(URLs.MNIST_SAMPLE)
|
path = untar_data(URLs.MNIST_SAMPLE)
|
||||||
|
@ -17,7 +17,8 @@ from tqdm import tqdm
|
|||||||
|
|
||||||
from clearml import Task, StorageManager
|
from clearml import Task, StorageManager
|
||||||
|
|
||||||
# ClearML Initializations
|
# Connecting ClearML with the current process,
|
||||||
|
# from here on everything is logged automatically
|
||||||
task = Task.init(project_name='Image Example', task_name='image classification CIFAR10')
|
task = Task.init(project_name='Image Example', task_name='image classification CIFAR10')
|
||||||
params = {'number_of_epochs': 20, 'batch_size': 64, 'dropout': 0.25, 'base_lr': 0.001, 'momentum': 0.9, 'loss_report': 100}
|
params = {'number_of_epochs': 20, 'batch_size': 64, 'dropout': 0.25, 'base_lr': 0.001, 'momentum': 0.9, 'loss_report': 100}
|
||||||
params = task.connect(params) # enabling configuration override by clearml
|
params = task.connect(params) # enabling configuration override by clearml
|
||||||
|
@ -89,7 +89,8 @@ model.compile(loss='categorical_crossentropy',
|
|||||||
optimizer=RMSprop(),
|
optimizer=RMSprop(),
|
||||||
metrics=['accuracy'])
|
metrics=['accuracy'])
|
||||||
|
|
||||||
# Connecting ClearML
|
# Connecting ClearML with the current process,
|
||||||
|
# from here on everything is logged automatically
|
||||||
task = Task.init(project_name='examples', task_name='Keras with TensorBoard example')
|
task = Task.init(project_name='examples', task_name='Keras with TensorBoard example')
|
||||||
|
|
||||||
# To set your own configuration:
|
# To set your own configuration:
|
||||||
|
@ -88,7 +88,8 @@ model.compile(loss='categorical_crossentropy',
|
|||||||
optimizer=RMSprop(),
|
optimizer=RMSprop(),
|
||||||
metrics=['accuracy'])
|
metrics=['accuracy'])
|
||||||
|
|
||||||
# Connecting ClearML
|
# Connecting ClearML with the current process,
|
||||||
|
# from here on everything is logged automatically
|
||||||
task = Task.init(project_name='examples', task_name='Keras with TensorBoard example')
|
task = Task.init(project_name='examples', task_name='Keras with TensorBoard example')
|
||||||
task.connect_configuration({'test': 1337, 'nested': {'key': 'value', 'number': 1}})
|
task.connect_configuration({'test': 1337, 'nested': {'key': 'value', 'number': 1}})
|
||||||
|
|
||||||
|
@ -7,7 +7,8 @@ from keras import Input, layers, Model
|
|||||||
|
|
||||||
from clearml import Task
|
from clearml import Task
|
||||||
|
|
||||||
|
# Connecting ClearML with the current process,
|
||||||
|
# from here on everything is logged automatically
|
||||||
task = Task.init(project_name='examples', task_name='Model configuration and upload')
|
task = Task.init(project_name='examples', task_name='Model configuration and upload')
|
||||||
|
|
||||||
|
|
||||||
|
@ -43,6 +43,8 @@ def build_model(hp):
|
|||||||
return model
|
return model
|
||||||
|
|
||||||
|
|
||||||
|
# Connecting ClearML with the current process,
|
||||||
|
# from here on everything is logged automatically
|
||||||
task = Task.init('examples', 'kerastuner cifar10 tuning')
|
task = Task.init('examples', 'kerastuner cifar10 tuning')
|
||||||
|
|
||||||
tuner = kt.Hyperband(
|
tuner = kt.Hyperband(
|
||||||
|
@ -6,6 +6,8 @@ from sklearn.metrics import mean_squared_error
|
|||||||
|
|
||||||
from clearml import Task
|
from clearml import Task
|
||||||
|
|
||||||
|
# Connecting ClearML with the current process,
|
||||||
|
# from here on everything is logged automatically
|
||||||
task = Task.init(project_name="examples", task_name="LIGHTgbm")
|
task = Task.init(project_name="examples", task_name="LIGHTgbm")
|
||||||
|
|
||||||
print('Loading data...')
|
print('Loading data...')
|
||||||
|
@ -5,7 +5,8 @@ import matplotlib.pyplot as plt
|
|||||||
import seaborn as sns
|
import seaborn as sns
|
||||||
from clearml import Task
|
from clearml import Task
|
||||||
|
|
||||||
|
# Connecting ClearML with the current process,
|
||||||
|
# from here on everything is logged automatically
|
||||||
task = Task.init(project_name='examples', task_name='Matplotlib example')
|
task = Task.init(project_name='examples', task_name='Matplotlib example')
|
||||||
|
|
||||||
# Create a plot
|
# Create a plot
|
||||||
|
@ -0,0 +1,95 @@
|
|||||||
|
from argparse import ArgumentParser
|
||||||
|
import torch
|
||||||
|
import pytorch_lightning as pl
|
||||||
|
from torch.nn import functional as F
|
||||||
|
from torch.utils.data import DataLoader, random_split
|
||||||
|
from clearml import Task
|
||||||
|
|
||||||
|
from torchvision.datasets.mnist import MNIST
|
||||||
|
from torchvision import transforms
|
||||||
|
|
||||||
|
|
||||||
|
class LitClassifier(pl.LightningModule):
|
||||||
|
def __init__(self, hidden_dim=128, learning_rate=1e-3):
|
||||||
|
super().__init__()
|
||||||
|
self.save_hyperparameters()
|
||||||
|
|
||||||
|
self.l1 = torch.nn.Linear(28 * 28, self.hparams.hidden_dim)
|
||||||
|
self.l2 = torch.nn.Linear(self.hparams.hidden_dim, 10)
|
||||||
|
|
||||||
|
def forward(self, x):
|
||||||
|
x = x.view(x.size(0), -1)
|
||||||
|
x = torch.relu(self.l1(x))
|
||||||
|
x = torch.relu(self.l2(x))
|
||||||
|
return x
|
||||||
|
|
||||||
|
def training_step(self, batch, batch_idx):
|
||||||
|
x, y = batch
|
||||||
|
y_hat = self(x)
|
||||||
|
loss = F.cross_entropy(y_hat, y)
|
||||||
|
return loss
|
||||||
|
|
||||||
|
def validation_step(self, batch, batch_idx):
|
||||||
|
x, y = batch
|
||||||
|
y_hat = self(x)
|
||||||
|
loss = F.cross_entropy(y_hat, y)
|
||||||
|
self.log('valid_loss', loss)
|
||||||
|
|
||||||
|
def test_step(self, batch, batch_idx):
|
||||||
|
x, y = batch
|
||||||
|
y_hat = self(x)
|
||||||
|
loss = F.cross_entropy(y_hat, y)
|
||||||
|
self.log('test_loss', loss)
|
||||||
|
|
||||||
|
def configure_optimizers(self):
|
||||||
|
return torch.optim.Adam(self.parameters(), lr=self.hparams.learning_rate)
|
||||||
|
|
||||||
|
@staticmethod
|
||||||
|
def add_model_specific_args(parent_parser):
|
||||||
|
parser = ArgumentParser(parents=[parent_parser], add_help=False)
|
||||||
|
parser.add_argument('--hidden_dim', type=int, default=128)
|
||||||
|
parser.add_argument('--learning_rate', type=float, default=0.0001)
|
||||||
|
return parser
|
||||||
|
|
||||||
|
|
||||||
|
if __name__ == '__main__':
|
||||||
|
# Connecting ClearML with the current process,
|
||||||
|
# from here on everything is logged automatically
|
||||||
|
task = Task.init(project_name="examples", task_name="pytorch lightning mnist example")
|
||||||
|
|
||||||
|
pl.seed_everything(0)
|
||||||
|
|
||||||
|
parser = ArgumentParser()
|
||||||
|
parser.add_argument('--batch_size', default=32, type=int)
|
||||||
|
parser.add_argument('--epochs', default=3, type=int)
|
||||||
|
parser = pl.Trainer.add_argparse_args(parser)
|
||||||
|
parser = LitClassifier.add_model_specific_args(parser)
|
||||||
|
args = parser.parse_args()
|
||||||
|
|
||||||
|
# ------------
|
||||||
|
# data
|
||||||
|
# ------------
|
||||||
|
dataset = MNIST('', train=True, download=True, transform=transforms.ToTensor())
|
||||||
|
mnist_test = MNIST('', train=False, download=True, transform=transforms.ToTensor())
|
||||||
|
mnist_train, mnist_val = random_split(dataset, [55000, 5000])
|
||||||
|
|
||||||
|
train_loader = DataLoader(mnist_train, batch_size=args.batch_size)
|
||||||
|
val_loader = DataLoader(mnist_val, batch_size=args.batch_size)
|
||||||
|
test_loader = DataLoader(mnist_test, batch_size=args.batch_size)
|
||||||
|
|
||||||
|
# ------------
|
||||||
|
# model
|
||||||
|
# ------------
|
||||||
|
model = LitClassifier(args.hidden_dim, args.learning_rate)
|
||||||
|
|
||||||
|
# ------------
|
||||||
|
# training
|
||||||
|
# ------------
|
||||||
|
trainer = pl.Trainer.from_argparse_args(args)
|
||||||
|
trainer.max_epochs = args.epochs
|
||||||
|
trainer.fit(model, train_loader, val_loader)
|
||||||
|
|
||||||
|
# ------------
|
||||||
|
# testing
|
||||||
|
# ------------
|
||||||
|
trainer.test(test_dataloaders=test_loader)
|
4
examples/frameworks/pytorch-lightning/requirements.txt
Normal file
4
examples/frameworks/pytorch-lightning/requirements.txt
Normal file
@ -0,0 +1,4 @@
|
|||||||
|
clearml
|
||||||
|
pytorch_lightning ~= 1.1.2
|
||||||
|
torch
|
||||||
|
torchvision
|
@ -6,7 +6,8 @@ from tempfile import gettempdir
|
|||||||
import torch
|
import torch
|
||||||
from clearml import Task
|
from clearml import Task
|
||||||
|
|
||||||
|
# Connecting ClearML with the current process,
|
||||||
|
# from here on everything is logged automatically
|
||||||
task = Task.init(project_name='examples', task_name='Model configuration and upload')
|
task = Task.init(project_name='examples', task_name='Model configuration and upload')
|
||||||
|
|
||||||
# create a model
|
# create a model
|
||||||
|
@ -62,7 +62,8 @@ import torchvision.models as models
|
|||||||
import copy
|
import copy
|
||||||
from clearml import Task
|
from clearml import Task
|
||||||
|
|
||||||
|
# Connecting ClearML with the current process,
|
||||||
|
# from here on everything is logged automatically
|
||||||
task = Task.init(project_name='examples', task_name='pytorch with matplotlib example', task_type=Task.TaskTypes.testing)
|
task = Task.init(project_name='examples', task_name='pytorch with matplotlib example', task_type=Task.TaskTypes.testing)
|
||||||
|
|
||||||
|
|
||||||
|
@ -74,6 +74,8 @@ def test(args, model, device, test_loader, epoch):
|
|||||||
|
|
||||||
|
|
||||||
def main():
|
def main():
|
||||||
|
# Connecting ClearML with the current process,
|
||||||
|
# from here on everything is logged automatically
|
||||||
task = Task.init(project_name='examples', task_name='pytorch mnist train')
|
task = Task.init(project_name='examples', task_name='pytorch mnist train')
|
||||||
|
|
||||||
# Training settings
|
# Training settings
|
||||||
|
@ -99,7 +99,11 @@ def main():
|
|||||||
parser.add_argument('--log-interval', type=int, default=10, metavar='N',
|
parser.add_argument('--log-interval', type=int, default=10, metavar='N',
|
||||||
help='how many batches to wait before logging training status')
|
help='how many batches to wait before logging training status')
|
||||||
args = parser.parse_args()
|
args = parser.parse_args()
|
||||||
Task.init(project_name='examples', task_name='pytorch with tensorboard')
|
|
||||||
|
# Connecting ClearML with the current process,
|
||||||
|
# from here on everything is logged automatically
|
||||||
|
task = Task.init(project_name='examples', task_name='pytorch with tensorboard') # noqa: F841
|
||||||
|
|
||||||
writer = SummaryWriter('runs')
|
writer = SummaryWriter('runs')
|
||||||
writer.add_text('TEXT', 'This is some text', 0)
|
writer.add_text('TEXT', 'This is some text', 0)
|
||||||
args.cuda = not args.no_cuda and torch.cuda.is_available()
|
args.cuda = not args.no_cuda and torch.cuda.is_available()
|
||||||
|
@ -6,9 +6,12 @@ from PIL import Image
|
|||||||
from torch.utils.tensorboard import SummaryWriter
|
from torch.utils.tensorboard import SummaryWriter
|
||||||
|
|
||||||
from clearml import Task
|
from clearml import Task
|
||||||
task = Task.init(project_name='examples', task_name='pytorch tensorboard toy example')
|
|
||||||
|
|
||||||
|
|
||||||
|
# Connecting ClearML with the current process,
|
||||||
|
# from here on everything is logged automatically
|
||||||
|
task = Task.init(project_name='examples', task_name='pytorch tensorboard toy example')
|
||||||
|
|
||||||
writer = SummaryWriter(log_dir=os.path.join(gettempdir(), 'tensorboard_logs'))
|
writer = SummaryWriter(log_dir=os.path.join(gettempdir(), 'tensorboard_logs'))
|
||||||
|
|
||||||
# convert to 4d [batch, col, row, RGB-channels]
|
# convert to 4d [batch, col, row, RGB-channels]
|
||||||
|
@ -9,9 +9,11 @@ from sklearn.model_selection import train_test_split
|
|||||||
import numpy as np
|
import numpy as np
|
||||||
import matplotlib.pyplot as plt
|
import matplotlib.pyplot as plt
|
||||||
|
|
||||||
|
|
||||||
from clearml import Task
|
from clearml import Task
|
||||||
|
|
||||||
|
|
||||||
|
# Connecting ClearML with the current process,
|
||||||
|
# from here on everything is logged automatically
|
||||||
task = Task.init(project_name="examples", task_name="scikit-learn joblib example")
|
task = Task.init(project_name="examples", task_name="scikit-learn joblib example")
|
||||||
|
|
||||||
iris = datasets.load_iris()
|
iris = datasets.load_iris()
|
||||||
|
@ -124,6 +124,8 @@ def plot_learning_curve(estimator, title, X, y, axes=None, ylim=None, cv=None, n
|
|||||||
return plt
|
return plt
|
||||||
|
|
||||||
|
|
||||||
|
# Connecting ClearML with the current process,
|
||||||
|
# from here on everything is logged automatically
|
||||||
Task.init('examples', 'scikit-learn matplotlib example')
|
Task.init('examples', 'scikit-learn matplotlib example')
|
||||||
|
|
||||||
fig, fig_axes = plt.subplots(1, 3, figsize=(30, 10))
|
fig, fig_axes = plt.subplots(1, 3, figsize=(30, 10))
|
||||||
|
@ -100,7 +100,10 @@ def main():
|
|||||||
args = parser.parse_args()
|
args = parser.parse_args()
|
||||||
args.cuda = not args.no_cuda and torch.cuda.is_available()
|
args.cuda = not args.no_cuda and torch.cuda.is_available()
|
||||||
|
|
||||||
task = Task.init(project_name='examples', task_name='pytorch with tensorboardX') # noqa: F841
|
# Connecting ClearML with the current process,
|
||||||
|
# from here on everything is logged automatically
|
||||||
|
task = Task.init(project_name='examples', task_name='pytorch with tensorboardX')
|
||||||
|
|
||||||
writer = SummaryWriter('runs')
|
writer = SummaryWriter('runs')
|
||||||
writer.add_text('TEXT', 'This is some text', 0)
|
writer.add_text('TEXT', 'This is some text', 0)
|
||||||
|
|
||||||
|
@ -39,6 +39,9 @@ import tensorflow as tf
|
|||||||
from tensorboard.plugins.pr_curve import summary
|
from tensorboard.plugins.pr_curve import summary
|
||||||
from clearml import Task
|
from clearml import Task
|
||||||
|
|
||||||
|
|
||||||
|
# Connecting ClearML with the current process,
|
||||||
|
# from here on everything is logged automatically
|
||||||
task = Task.init(project_name='examples', task_name='tensorboard pr_curve')
|
task = Task.init(project_name='examples', task_name='tensorboard pr_curve')
|
||||||
|
|
||||||
tf.compat.v1.disable_v2_behavior()
|
tf.compat.v1.disable_v2_behavior()
|
||||||
|
@ -8,9 +8,12 @@ import numpy as np
|
|||||||
from PIL import Image
|
from PIL import Image
|
||||||
|
|
||||||
from clearml import Task
|
from clearml import Task
|
||||||
task = Task.init(project_name='examples', task_name='tensorboard toy example')
|
|
||||||
|
|
||||||
|
|
||||||
|
# Connecting ClearML with the current process,
|
||||||
|
# from here on everything is logged automatically
|
||||||
|
task = Task.init(project_name='examples', task_name='tensorboard toy example')
|
||||||
|
|
||||||
k = tf.placeholder(tf.float32)
|
k = tf.placeholder(tf.float32)
|
||||||
|
|
||||||
# Make a normal distribution, with a shifting mean
|
# Make a normal distribution, with a shifting mean
|
||||||
|
@ -32,11 +32,13 @@ from tensorflow.examples.tutorials.mnist import input_data
|
|||||||
|
|
||||||
from clearml import Task
|
from clearml import Task
|
||||||
|
|
||||||
|
|
||||||
tf.compat.v1.enable_eager_execution()
|
tf.compat.v1.enable_eager_execution()
|
||||||
|
|
||||||
|
# Connecting ClearML with the current process,
|
||||||
|
# from here on everything is logged automatically
|
||||||
task = Task.init(project_name='examples', task_name='Tensorflow eager mode')
|
task = Task.init(project_name='examples', task_name='Tensorflow eager mode')
|
||||||
|
|
||||||
|
|
||||||
FLAGS = tf.app.flags.FLAGS
|
FLAGS = tf.app.flags.FLAGS
|
||||||
tf.app.flags.DEFINE_integer('data_num', 100, """Flag of type integer""")
|
tf.app.flags.DEFINE_integer('data_num', 100, """Flag of type integer""")
|
||||||
tf.app.flags.DEFINE_string('img_path', './img', """Flag of type string""")
|
tf.app.flags.DEFINE_string('img_path', './img', """Flag of type string""")
|
||||||
|
@ -34,6 +34,9 @@ from tensorflow.examples.tutorials.mnist import input_data
|
|||||||
from clearml import Task
|
from clearml import Task
|
||||||
|
|
||||||
FLAGS = None
|
FLAGS = None
|
||||||
|
|
||||||
|
# Connecting ClearML with the current process,
|
||||||
|
# from here on everything is logged automatically
|
||||||
task = Task.init(project_name='examples', task_name='Tensorflow mnist with summaries example')
|
task = Task.init(project_name='examples', task_name='Tensorflow mnist with summaries example')
|
||||||
|
|
||||||
|
|
||||||
|
@ -6,6 +6,9 @@ import tempfile
|
|||||||
import tensorflow as tf
|
import tensorflow as tf
|
||||||
from clearml import Task
|
from clearml import Task
|
||||||
|
|
||||||
|
|
||||||
|
# Connecting ClearML with the current process,
|
||||||
|
# from here on everything is logged automatically
|
||||||
task = Task.init(project_name='examples', task_name='Model configuration and upload')
|
task = Task.init(project_name='examples', task_name='Model configuration and upload')
|
||||||
|
|
||||||
model = tf.Module()
|
model = tf.Module()
|
||||||
|
@ -39,6 +39,8 @@ from tensorboard.plugins.pr_curve import summary
|
|||||||
from clearml import Task
|
from clearml import Task
|
||||||
|
|
||||||
|
|
||||||
|
# Connecting ClearML with the current process,
|
||||||
|
# from here on everything is logged automatically
|
||||||
task = Task.init(project_name='examples', task_name='tensorboard pr_curve')
|
task = Task.init(project_name='examples', task_name='tensorboard pr_curve')
|
||||||
|
|
||||||
tf.compat.v1.disable_v2_behavior()
|
tf.compat.v1.disable_v2_behavior()
|
||||||
|
@ -11,8 +11,9 @@ from tensorflow.keras import Model
|
|||||||
from clearml import Task
|
from clearml import Task
|
||||||
|
|
||||||
|
|
||||||
task = Task.init(project_name='examples',
|
# Connecting ClearML with the current process,
|
||||||
task_name='Tensorflow v2 mnist with summaries')
|
# from here on everything is logged automatically
|
||||||
|
task = Task.init(project_name='examples', task_name='Tensorflow v2 mnist with summaries')
|
||||||
|
|
||||||
|
|
||||||
# Load and prepare the MNIST dataset.
|
# Load and prepare the MNIST dataset.
|
||||||
|
@ -7,7 +7,11 @@ from xgboost import plot_tree
|
|||||||
|
|
||||||
from clearml import Task
|
from clearml import Task
|
||||||
|
|
||||||
|
|
||||||
|
# Connecting ClearML with the current process,
|
||||||
|
# from here on everything is logged automatically
|
||||||
task = Task.init(project_name='examples', task_name='XGBoost simple example')
|
task = Task.init(project_name='examples', task_name='XGBoost simple example')
|
||||||
|
|
||||||
iris = datasets.load_iris()
|
iris = datasets.load_iris()
|
||||||
X = iris.data
|
X = iris.data
|
||||||
y = iris.target
|
y = iris.target
|
||||||
@ -56,5 +60,6 @@ labels = dtest.get_label()
|
|||||||
|
|
||||||
# plot results
|
# plot results
|
||||||
xgb.plot_importance(model)
|
xgb.plot_importance(model)
|
||||||
|
plt.show()
|
||||||
plot_tree(model)
|
plot_tree(model)
|
||||||
plt.show()
|
plt.show()
|
||||||
|
@ -20,7 +20,8 @@ from tensorflow.keras.optimizers import RMSprop
|
|||||||
from clearml import Task, Logger
|
from clearml import Task, Logger
|
||||||
|
|
||||||
|
|
||||||
# Connecting ClearML
|
# Connecting ClearML with the current process,
|
||||||
|
# from here on everything is logged automatically
|
||||||
task = Task.init(project_name='examples', task_name='Keras HP optimization base')
|
task = Task.init(project_name='examples', task_name='Keras HP optimization base')
|
||||||
|
|
||||||
|
|
||||||
|
@ -40,7 +40,8 @@ def job_complete_callback(
|
|||||||
print('WOOT WOOT we broke the record! Objective reached {}'.format(objective_value))
|
print('WOOT WOOT we broke the record! Objective reached {}'.format(objective_value))
|
||||||
|
|
||||||
|
|
||||||
# Connecting ClearML
|
# Connecting ClearML with the current process,
|
||||||
|
# from here on everything is logged automatically
|
||||||
task = Task.init(project_name='Hyper-Parameter Optimization',
|
task = Task.init(project_name='Hyper-Parameter Optimization',
|
||||||
task_name='Automatic Hyper-Parameter Optimization',
|
task_name='Automatic Hyper-Parameter Optimization',
|
||||||
task_type=Task.TaskTypes.optimizer,
|
task_type=Task.TaskTypes.optimizer,
|
||||||
@ -73,11 +74,10 @@ an_optimizer = HyperParameterOptimizer(
|
|||||||
UniformIntegerParameterRange('General/layer_2', min_value=128, max_value=512, step_size=128),
|
UniformIntegerParameterRange('General/layer_2', min_value=128, max_value=512, step_size=128),
|
||||||
DiscreteParameterRange('General/batch_size', values=[96, 128, 160]),
|
DiscreteParameterRange('General/batch_size', values=[96, 128, 160]),
|
||||||
DiscreteParameterRange('General/epochs', values=[30]),
|
DiscreteParameterRange('General/epochs', values=[30]),
|
||||||
DiscreteParameterRange('General/optimizer', values=['adam', 'sgd']),
|
|
||||||
],
|
],
|
||||||
# this is the objective metric we want to maximize/minimize
|
# this is the objective metric we want to maximize/minimize
|
||||||
objective_metric_title='accuracy',
|
objective_metric_title='epoch_accuracy',
|
||||||
objective_metric_series='accuracy',
|
objective_metric_series='epoch_accuracy',
|
||||||
# now we decide if we want to maximize it or minimize it (accuracy we maximize)
|
# now we decide if we want to maximize it or minimize it (accuracy we maximize)
|
||||||
objective_metric_sign='max',
|
objective_metric_sign='max',
|
||||||
# let us limit the number of concurrent experiments,
|
# let us limit the number of concurrent experiments,
|
||||||
|
@ -2,6 +2,8 @@ from clearml import Task
|
|||||||
from clearml.automation.controller import PipelineController
|
from clearml.automation.controller import PipelineController
|
||||||
|
|
||||||
|
|
||||||
|
# Connecting ClearML with the current process,
|
||||||
|
# from here on everything is logged automatically
|
||||||
task = Task.init(project_name='examples', task_name='pipeline demo',
|
task = Task.init(project_name='examples', task_name='pipeline demo',
|
||||||
task_type=Task.TaskTypes.controller, reuse_last_task_id=False)
|
task_type=Task.TaskTypes.controller, reuse_last_task_id=False)
|
||||||
|
|
||||||
|
@ -3,7 +3,8 @@ from clearml import Task, StorageManager
|
|||||||
from sklearn.model_selection import train_test_split
|
from sklearn.model_selection import train_test_split
|
||||||
|
|
||||||
|
|
||||||
# Connecting ClearML
|
# Connecting ClearML with the current process,
|
||||||
|
# from here on everything is logged automatically
|
||||||
task = Task.init(project_name="examples", task_name="pipeline step 2 process dataset")
|
task = Task.init(project_name="examples", task_name="pipeline step 2 process dataset")
|
||||||
|
|
||||||
# program arguments
|
# program arguments
|
||||||
|
@ -5,7 +5,9 @@ from sklearn.linear_model import LogisticRegression
|
|||||||
|
|
||||||
from clearml import Task
|
from clearml import Task
|
||||||
|
|
||||||
# Connecting ClearML
|
|
||||||
|
# Connecting ClearML with the current process,
|
||||||
|
# from here on everything is logged automatically
|
||||||
task = Task.init(project_name="examples", task_name="pipeline step 3 train model")
|
task = Task.init(project_name="examples", task_name="pipeline step 3 train model")
|
||||||
|
|
||||||
# Arguments
|
# Arguments
|
||||||
|
@ -39,7 +39,8 @@ def report_plots(logger, iteration=0):
|
|||||||
|
|
||||||
|
|
||||||
def main():
|
def main():
|
||||||
# Create the experiment Task
|
# Connecting ClearML with the current process,
|
||||||
|
# from here on everything is logged automatically
|
||||||
task = Task.init(project_name="examples", task_name="3D plot reporting")
|
task = Task.init(project_name="examples", task_name="3D plot reporting")
|
||||||
|
|
||||||
print('reporting 3D plot graphs')
|
print('reporting 3D plot graphs')
|
||||||
|
@ -6,7 +6,10 @@ import numpy as np
|
|||||||
from PIL import Image
|
from PIL import Image
|
||||||
from clearml import Task
|
from clearml import Task
|
||||||
|
|
||||||
task = Task.init('examples', 'artifacts example')
|
|
||||||
|
# Connecting ClearML with the current process,
|
||||||
|
# from here on everything is logged automatically
|
||||||
|
task = Task.init(project_name='examples', task_name='artifacts example')
|
||||||
|
|
||||||
df = pd.DataFrame(
|
df = pd.DataFrame(
|
||||||
{
|
{
|
||||||
|
@ -217,7 +217,8 @@ def report_html_image(logger, iteration=0):
|
|||||||
|
|
||||||
|
|
||||||
def main():
|
def main():
|
||||||
# Create the experiment Task
|
# Connecting ClearML with the current process,
|
||||||
|
# from here on everything is logged automatically
|
||||||
task = Task.init(project_name="examples", task_name="html samples reporting")
|
task = Task.init(project_name="examples", task_name="html samples reporting")
|
||||||
|
|
||||||
print('reporting html files into debug samples section')
|
print('reporting html files into debug samples section')
|
||||||
|
@ -19,6 +19,8 @@ FLAGS = flags.FLAGS
|
|||||||
flags.DEFINE_string('echo', None, 'Text to echo.')
|
flags.DEFINE_string('echo', None, 'Text to echo.')
|
||||||
flags.DEFINE_string('another_str', 'My string', 'A string', module_name='test')
|
flags.DEFINE_string('another_str', 'My string', 'A string', module_name='test')
|
||||||
|
|
||||||
|
# Connecting ClearML with the current process,
|
||||||
|
# from here on everything is logged automatically
|
||||||
task = Task.init(project_name='examples', task_name='hyper-parameters example')
|
task = Task.init(project_name='examples', task_name='hyper-parameters example')
|
||||||
|
|
||||||
flags.DEFINE_integer('echo3', 3, 'Text to echo.')
|
flags.DEFINE_integer('echo3', 3, 'Text to echo.')
|
||||||
@ -32,8 +34,7 @@ parameters = {
|
|||||||
'float': 2.2,
|
'float': 2.2,
|
||||||
'string': 'my string',
|
'string': 'my string',
|
||||||
}
|
}
|
||||||
from clearml import Task
|
parameters = task.connect(parameters)
|
||||||
parameters = Task.current_task().connect(parameters, name='more_stuff_deep_inside_code')
|
|
||||||
|
|
||||||
# adding new parameter after connect (will be logged as well)
|
# adding new parameter after connect (will be logged as well)
|
||||||
parameters['new_param'] = 'this is new'
|
parameters['new_param'] = 'this is new'
|
||||||
|
@ -43,7 +43,8 @@ def report_debug_images(logger, iteration=0):
|
|||||||
|
|
||||||
|
|
||||||
def main():
|
def main():
|
||||||
# Create the experiment Task
|
# Connecting ClearML with the current process,
|
||||||
|
# from here on everything is logged automatically
|
||||||
task = Task.init(project_name="examples", task_name="image reporting")
|
task = Task.init(project_name="examples", task_name="image reporting")
|
||||||
|
|
||||||
print('reporting a few debug images')
|
print('reporting a few debug images')
|
||||||
|
@ -4,6 +4,8 @@ import numpy as np
|
|||||||
import matplotlib.pyplot as plt
|
import matplotlib.pyplot as plt
|
||||||
from clearml import Task
|
from clearml import Task
|
||||||
|
|
||||||
|
# Connecting ClearML with the current process,
|
||||||
|
# from here on everything is logged automatically
|
||||||
# Create a new task, disable automatic matplotlib connect
|
# Create a new task, disable automatic matplotlib connect
|
||||||
task = Task.init(
|
task = Task.init(
|
||||||
project_name='examples',
|
project_name='examples',
|
||||||
|
@ -4,6 +4,8 @@ import os
|
|||||||
from clearml import Task, Logger
|
from clearml import Task, Logger
|
||||||
|
|
||||||
|
|
||||||
|
# Connecting ClearML with the current process,
|
||||||
|
# from here on everything is logged automatically
|
||||||
task = Task.init(project_name="examples", task_name="audio and video reporting")
|
task = Task.init(project_name="examples", task_name="audio and video reporting")
|
||||||
|
|
||||||
print('reporting audio and video samples to the debug samples section')
|
print('reporting audio and video samples to the debug samples section')
|
||||||
|
@ -5,6 +5,8 @@ import os
|
|||||||
from clearml import Task, OutputModel
|
from clearml import Task, OutputModel
|
||||||
|
|
||||||
|
|
||||||
|
# Connecting ClearML with the current process,
|
||||||
|
# from here on everything is logged automatically
|
||||||
task = Task.init(project_name='examples', task_name='Model configuration example')
|
task = Task.init(project_name='examples', task_name='Model configuration example')
|
||||||
|
|
||||||
# Connect a local configuration file
|
# Connect a local configuration file
|
||||||
|
@ -43,7 +43,8 @@ def report_table(logger, iteration=0):
|
|||||||
|
|
||||||
|
|
||||||
def main():
|
def main():
|
||||||
# Create the experiment Task
|
# Connecting ClearML with the current process,
|
||||||
|
# from here on everything is logged automatically
|
||||||
task = Task.init(project_name="examples", task_name="table reporting")
|
task = Task.init(project_name="examples", task_name="table reporting")
|
||||||
|
|
||||||
print('reporting pandas tables and python lists as tables into the plots section')
|
print('reporting pandas tables and python lists as tables into the plots section')
|
||||||
|
@ -4,6 +4,8 @@ from clearml import Task
|
|||||||
import plotly.express as px
|
import plotly.express as px
|
||||||
|
|
||||||
|
|
||||||
|
# Connecting ClearML with the current process,
|
||||||
|
# from here on everything is logged automatically
|
||||||
task = Task.init('examples', 'plotly reporting')
|
task = Task.init('examples', 'plotly reporting')
|
||||||
|
|
||||||
print('reporting plotly figures')
|
print('reporting plotly figures')
|
||||||
|
@ -21,7 +21,8 @@ def report_scalars(logger):
|
|||||||
|
|
||||||
|
|
||||||
def main():
|
def main():
|
||||||
# Create the experiment Task
|
# Connecting ClearML with the current process,
|
||||||
|
# from here on everything is logged automatically
|
||||||
task = Task.init(project_name="examples", task_name="scalar reporting")
|
task = Task.init(project_name="examples", task_name="scalar reporting")
|
||||||
|
|
||||||
print('reporting scalar graphs')
|
print('reporting scalar graphs')
|
||||||
|
@ -109,7 +109,8 @@ def report_plots(logger, iteration=0):
|
|||||||
|
|
||||||
|
|
||||||
def main():
|
def main():
|
||||||
# Create the experiment Task
|
# Connecting ClearML with the current process,
|
||||||
|
# from here on everything is logged automatically
|
||||||
task = Task.init(project_name="examples", task_name="2D plots reporting")
|
task = Task.init(project_name="examples", task_name="2D plots reporting")
|
||||||
|
|
||||||
print('reporting some graphs')
|
print('reporting some graphs')
|
||||||
@ -117,7 +118,6 @@ def main():
|
|||||||
# Get the task logger,
|
# Get the task logger,
|
||||||
# You can also call Task.current_task().get_logger() from anywhere in your code.
|
# You can also call Task.current_task().get_logger() from anywhere in your code.
|
||||||
logger = task.get_logger()
|
logger = task.get_logger()
|
||||||
#logger.report_scatter2d()
|
|
||||||
|
|
||||||
# report graphs
|
# report graphs
|
||||||
report_plots(logger)
|
report_plots(logger)
|
||||||
|
@ -58,7 +58,8 @@ Vestibulum dictum ipsum at viverra ultrices. Aliquam sed ante massa. Quisque con
|
|||||||
|
|
||||||
|
|
||||||
def main():
|
def main():
|
||||||
# Create the experiment Task
|
# Connecting ClearML with the current process,
|
||||||
|
# from here on everything is logged automatically
|
||||||
task = Task.init(project_name="examples", task_name="text reporting")
|
task = Task.init(project_name="examples", task_name="text reporting")
|
||||||
|
|
||||||
print("reporting text logs")
|
print("reporting text logs")
|
||||||
|
@ -76,6 +76,8 @@ def main():
|
|||||||
)
|
)
|
||||||
return
|
return
|
||||||
|
|
||||||
|
# Connecting ClearML with the current process,
|
||||||
|
# from here on everything is logged automatically
|
||||||
task = Task.init(project_name="DevOps", task_name="AWS Auto-Scaler", task_type=Task.TaskTypes.service)
|
task = Task.init(project_name="DevOps", task_name="AWS Auto-Scaler", task_type=Task.TaskTypes.service)
|
||||||
task.connect(hyper_params)
|
task.connect(hyper_params)
|
||||||
task.connect_configuration(configurations)
|
task.connect_configuration(configurations)
|
||||||
|
@ -24,7 +24,8 @@ from clearml.backend_api.session.client import APIClient
|
|||||||
|
|
||||||
from clearml import Task
|
from clearml import Task
|
||||||
|
|
||||||
# Connecting ClearML
|
# Connecting ClearML with the current process,
|
||||||
|
# from here on everything is logged automatically
|
||||||
task = Task.init(
|
task = Task.init(
|
||||||
project_name="DevOps",
|
project_name="DevOps",
|
||||||
task_name="Cleanup Service",
|
task_name="Cleanup Service",
|
||||||
|
@ -12,9 +12,13 @@ import jupyter # noqa
|
|||||||
from clearml import Task
|
from clearml import Task
|
||||||
|
|
||||||
|
|
||||||
# initialize ClearML
|
# Connecting ClearML with the current process,
|
||||||
|
# from here on everything is logged automatically
|
||||||
task = Task.init(
|
task = Task.init(
|
||||||
project_name="DevOps", task_name="Allocate Jupyter Notebook Instance", task_type=Task.TaskTypes.service)
|
project_name="DevOps",
|
||||||
|
task_name="Allocate Jupyter Notebook Instance",
|
||||||
|
task_type=Task.TaskTypes.service
|
||||||
|
)
|
||||||
|
|
||||||
# get rid of all the runtime ClearML
|
# get rid of all the runtime ClearML
|
||||||
preserve = (
|
preserve = (
|
||||||
|
@ -192,6 +192,8 @@ def main():
|
|||||||
slack_monitor.status_alerts += ["completed"]
|
slack_monitor.status_alerts += ["completed"]
|
||||||
|
|
||||||
# start the monitoring Task
|
# start the monitoring Task
|
||||||
|
# Connecting ClearML with the current process,
|
||||||
|
# from here on everything is logged automatically
|
||||||
task = Task.init(project_name='Monitoring', task_name='Slack Alerts', task_type=Task.TaskTypes.monitor)
|
task = Task.init(project_name='Monitoring', task_name='Slack Alerts', task_type=Task.TaskTypes.monitor)
|
||||||
if not args.local:
|
if not args.local:
|
||||||
task.execute_remotely(queue_name=args.service_queue)
|
task.execute_remotely(queue_name=args.service_queue)
|
||||||
|
Loading…
Reference in New Issue
Block a user