diff --git a/examples/automation/manual_random_param_search_example.py b/examples/automation/manual_random_param_search_example.py index 2bd7ec96..a54ad0ce 100644 --- a/examples/automation/manual_random_param_search_example.py +++ b/examples/automation/manual_random_param_search_example.py @@ -2,7 +2,8 @@ from random import sample from clearml import Task -# Connecting ClearML +# Connecting ClearML with the current process, +# from here on everything is logged automatically task = Task.init(project_name='examples', task_name='Random Hyper-Parameter Search Example', task_type=Task.TaskTypes.optimizer) # Create a hyper-parameter dictionary for the task diff --git a/examples/frameworks/autokeras/autokeras_imdb_example.py b/examples/frameworks/autokeras/autokeras_imdb_example.py index 6d268891..2df6e78f 100644 --- a/examples/frameworks/autokeras/autokeras_imdb_example.py +++ b/examples/frameworks/autokeras/autokeras_imdb_example.py @@ -5,6 +5,8 @@ from tensorflow import keras from clearml import Task +# Connecting ClearML with the current process, +# from here on everything is logged automatically task = Task.init(project_name="autokeras", task_name="autokeras imdb example with scalars") diff --git a/examples/frameworks/fastai/fastai_with_tensorboard.py b/examples/frameworks/fastai/fastai_with_tensorboard.py index 2fb2facd..570a9ca6 100644 --- a/examples/frameworks/fastai/fastai_with_tensorboard.py +++ b/examples/frameworks/fastai/fastai_with_tensorboard.py @@ -6,6 +6,8 @@ from fastai.vision import * # Quick access to computer vision functionality from clearml import Task +# Connecting ClearML with the current process, +# from here on everything is logged automatically task = Task.init(project_name="example", task_name="fastai with tensorboard callback") path = untar_data(URLs.MNIST_SAMPLE) diff --git a/examples/frameworks/ignite/cifar_ignite.py b/examples/frameworks/ignite/cifar_ignite.py index 0ebbfd89..79fc3a9f 100644 --- a/examples/frameworks/ignite/cifar_ignite.py +++ b/examples/frameworks/ignite/cifar_ignite.py @@ -17,7 +17,8 @@ from tqdm import tqdm from clearml import Task, StorageManager -# ClearML Initializations +# Connecting ClearML with the current process, +# from here on everything is logged automatically task = Task.init(project_name='Image Example', task_name='image classification CIFAR10') params = {'number_of_epochs': 20, 'batch_size': 64, 'dropout': 0.25, 'base_lr': 0.001, 'momentum': 0.9, 'loss_report': 100} params = task.connect(params) # enabling configuration override by clearml diff --git a/examples/frameworks/keras/keras_tensorboard.py b/examples/frameworks/keras/keras_tensorboard.py index d24cf1ed..6762965f 100644 --- a/examples/frameworks/keras/keras_tensorboard.py +++ b/examples/frameworks/keras/keras_tensorboard.py @@ -89,7 +89,8 @@ model.compile(loss='categorical_crossentropy', optimizer=RMSprop(), metrics=['accuracy']) -# Connecting ClearML +# Connecting ClearML with the current process, +# from here on everything is logged automatically task = Task.init(project_name='examples', task_name='Keras with TensorBoard example') # To set your own configuration: diff --git a/examples/frameworks/keras/legacy/keras_tensorboard.py b/examples/frameworks/keras/legacy/keras_tensorboard.py index 28f04d3d..c30cb0d2 100644 --- a/examples/frameworks/keras/legacy/keras_tensorboard.py +++ b/examples/frameworks/keras/legacy/keras_tensorboard.py @@ -88,7 +88,8 @@ model.compile(loss='categorical_crossentropy', optimizer=RMSprop(), metrics=['accuracy']) -# Connecting ClearML +# Connecting ClearML with the current process, +# from here on everything is logged automatically task = Task.init(project_name='examples', task_name='Keras with TensorBoard example') task.connect_configuration({'test': 1337, 'nested': {'key': 'value', 'number': 1}}) diff --git a/examples/frameworks/keras/manual_model_upload.py b/examples/frameworks/keras/manual_model_upload.py index 6b2a0b4f..11c8e6de 100644 --- a/examples/frameworks/keras/manual_model_upload.py +++ b/examples/frameworks/keras/manual_model_upload.py @@ -7,7 +7,8 @@ from keras import Input, layers, Model from clearml import Task - +# Connecting ClearML with the current process, +# from here on everything is logged automatically task = Task.init(project_name='examples', task_name='Model configuration and upload') diff --git a/examples/frameworks/kerastuner/keras_tuner_cifar.py b/examples/frameworks/kerastuner/keras_tuner_cifar.py index a2b5e019..9381f7ef 100644 --- a/examples/frameworks/kerastuner/keras_tuner_cifar.py +++ b/examples/frameworks/kerastuner/keras_tuner_cifar.py @@ -43,6 +43,8 @@ def build_model(hp): return model +# Connecting ClearML with the current process, +# from here on everything is logged automatically task = Task.init('examples', 'kerastuner cifar10 tuning') tuner = kt.Hyperband( diff --git a/examples/frameworks/lightgbm/train_with_lightbgm.py b/examples/frameworks/lightgbm/train_with_lightbgm.py index b6fad561..c12a4a24 100644 --- a/examples/frameworks/lightgbm/train_with_lightbgm.py +++ b/examples/frameworks/lightgbm/train_with_lightbgm.py @@ -6,6 +6,8 @@ from sklearn.metrics import mean_squared_error from clearml import Task +# Connecting ClearML with the current process, +# from here on everything is logged automatically task = Task.init(project_name="examples", task_name="LIGHTgbm") print('Loading data...') diff --git a/examples/frameworks/matplotlib/matplotlib_example.py b/examples/frameworks/matplotlib/matplotlib_example.py index 47575937..ebc6cedb 100644 --- a/examples/frameworks/matplotlib/matplotlib_example.py +++ b/examples/frameworks/matplotlib/matplotlib_example.py @@ -5,7 +5,8 @@ import matplotlib.pyplot as plt import seaborn as sns from clearml import Task - +# Connecting ClearML with the current process, +# from here on everything is logged automatically task = Task.init(project_name='examples', task_name='Matplotlib example') # Create a plot diff --git a/examples/frameworks/pytorch-lightning/pytorch_lightning_example.py b/examples/frameworks/pytorch-lightning/pytorch_lightning_example.py new file mode 100644 index 00000000..fa78ec5a --- /dev/null +++ b/examples/frameworks/pytorch-lightning/pytorch_lightning_example.py @@ -0,0 +1,95 @@ +from argparse import ArgumentParser +import torch +import pytorch_lightning as pl +from torch.nn import functional as F +from torch.utils.data import DataLoader, random_split +from clearml import Task + +from torchvision.datasets.mnist import MNIST +from torchvision import transforms + + +class LitClassifier(pl.LightningModule): + def __init__(self, hidden_dim=128, learning_rate=1e-3): + super().__init__() + self.save_hyperparameters() + + self.l1 = torch.nn.Linear(28 * 28, self.hparams.hidden_dim) + self.l2 = torch.nn.Linear(self.hparams.hidden_dim, 10) + + def forward(self, x): + x = x.view(x.size(0), -1) + x = torch.relu(self.l1(x)) + x = torch.relu(self.l2(x)) + return x + + def training_step(self, batch, batch_idx): + x, y = batch + y_hat = self(x) + loss = F.cross_entropy(y_hat, y) + return loss + + def validation_step(self, batch, batch_idx): + x, y = batch + y_hat = self(x) + loss = F.cross_entropy(y_hat, y) + self.log('valid_loss', loss) + + def test_step(self, batch, batch_idx): + x, y = batch + y_hat = self(x) + loss = F.cross_entropy(y_hat, y) + self.log('test_loss', loss) + + def configure_optimizers(self): + return torch.optim.Adam(self.parameters(), lr=self.hparams.learning_rate) + + @staticmethod + def add_model_specific_args(parent_parser): + parser = ArgumentParser(parents=[parent_parser], add_help=False) + parser.add_argument('--hidden_dim', type=int, default=128) + parser.add_argument('--learning_rate', type=float, default=0.0001) + return parser + + +if __name__ == '__main__': + # Connecting ClearML with the current process, + # from here on everything is logged automatically + task = Task.init(project_name="examples", task_name="pytorch lightning mnist example") + + pl.seed_everything(0) + + parser = ArgumentParser() + parser.add_argument('--batch_size', default=32, type=int) + parser.add_argument('--epochs', default=3, type=int) + parser = pl.Trainer.add_argparse_args(parser) + parser = LitClassifier.add_model_specific_args(parser) + args = parser.parse_args() + + # ------------ + # data + # ------------ + dataset = MNIST('', train=True, download=True, transform=transforms.ToTensor()) + mnist_test = MNIST('', train=False, download=True, transform=transforms.ToTensor()) + mnist_train, mnist_val = random_split(dataset, [55000, 5000]) + + train_loader = DataLoader(mnist_train, batch_size=args.batch_size) + val_loader = DataLoader(mnist_val, batch_size=args.batch_size) + test_loader = DataLoader(mnist_test, batch_size=args.batch_size) + + # ------------ + # model + # ------------ + model = LitClassifier(args.hidden_dim, args.learning_rate) + + # ------------ + # training + # ------------ + trainer = pl.Trainer.from_argparse_args(args) + trainer.max_epochs = args.epochs + trainer.fit(model, train_loader, val_loader) + + # ------------ + # testing + # ------------ + trainer.test(test_dataloaders=test_loader) diff --git a/examples/frameworks/pytorch-lightning/requirements.txt b/examples/frameworks/pytorch-lightning/requirements.txt new file mode 100644 index 00000000..cd407d0e --- /dev/null +++ b/examples/frameworks/pytorch-lightning/requirements.txt @@ -0,0 +1,4 @@ +clearml +pytorch_lightning ~= 1.1.2 +torch +torchvision diff --git a/examples/frameworks/pytorch/manual_model_upload.py b/examples/frameworks/pytorch/manual_model_upload.py index 485d33fb..a6f6a986 100644 --- a/examples/frameworks/pytorch/manual_model_upload.py +++ b/examples/frameworks/pytorch/manual_model_upload.py @@ -6,7 +6,8 @@ from tempfile import gettempdir import torch from clearml import Task - +# Connecting ClearML with the current process, +# from here on everything is logged automatically task = Task.init(project_name='examples', task_name='Model configuration and upload') # create a model diff --git a/examples/frameworks/pytorch/pytorch_matplotlib.py b/examples/frameworks/pytorch/pytorch_matplotlib.py index 53a88b00..b51216f7 100644 --- a/examples/frameworks/pytorch/pytorch_matplotlib.py +++ b/examples/frameworks/pytorch/pytorch_matplotlib.py @@ -62,7 +62,8 @@ import torchvision.models as models import copy from clearml import Task - +# Connecting ClearML with the current process, +# from here on everything is logged automatically task = Task.init(project_name='examples', task_name='pytorch with matplotlib example', task_type=Task.TaskTypes.testing) diff --git a/examples/frameworks/pytorch/pytorch_mnist.py b/examples/frameworks/pytorch/pytorch_mnist.py index b1fac4e6..885a490c 100644 --- a/examples/frameworks/pytorch/pytorch_mnist.py +++ b/examples/frameworks/pytorch/pytorch_mnist.py @@ -74,6 +74,8 @@ def test(args, model, device, test_loader, epoch): def main(): + # Connecting ClearML with the current process, + # from here on everything is logged automatically task = Task.init(project_name='examples', task_name='pytorch mnist train') # Training settings diff --git a/examples/frameworks/pytorch/pytorch_tensorboard.py b/examples/frameworks/pytorch/pytorch_tensorboard.py index 58005848..6aca2fbd 100644 --- a/examples/frameworks/pytorch/pytorch_tensorboard.py +++ b/examples/frameworks/pytorch/pytorch_tensorboard.py @@ -99,7 +99,11 @@ def main(): parser.add_argument('--log-interval', type=int, default=10, metavar='N', help='how many batches to wait before logging training status') args = parser.parse_args() - Task.init(project_name='examples', task_name='pytorch with tensorboard') + + # Connecting ClearML with the current process, + # from here on everything is logged automatically + task = Task.init(project_name='examples', task_name='pytorch with tensorboard') # noqa: F841 + writer = SummaryWriter('runs') writer.add_text('TEXT', 'This is some text', 0) args.cuda = not args.no_cuda and torch.cuda.is_available() diff --git a/examples/frameworks/pytorch/tensorboard_toy_pytorch.py b/examples/frameworks/pytorch/tensorboard_toy_pytorch.py index 8ff1dc75..2eb08425 100644 --- a/examples/frameworks/pytorch/tensorboard_toy_pytorch.py +++ b/examples/frameworks/pytorch/tensorboard_toy_pytorch.py @@ -6,9 +6,12 @@ from PIL import Image from torch.utils.tensorboard import SummaryWriter from clearml import Task -task = Task.init(project_name='examples', task_name='pytorch tensorboard toy example') +# Connecting ClearML with the current process, +# from here on everything is logged automatically +task = Task.init(project_name='examples', task_name='pytorch tensorboard toy example') + writer = SummaryWriter(log_dir=os.path.join(gettempdir(), 'tensorboard_logs')) # convert to 4d [batch, col, row, RGB-channels] diff --git a/examples/frameworks/scikit-learn/sklearn_joblib_example.py b/examples/frameworks/scikit-learn/sklearn_joblib_example.py index b29a6bef..7edae99e 100644 --- a/examples/frameworks/scikit-learn/sklearn_joblib_example.py +++ b/examples/frameworks/scikit-learn/sklearn_joblib_example.py @@ -9,9 +9,11 @@ from sklearn.model_selection import train_test_split import numpy as np import matplotlib.pyplot as plt - from clearml import Task + +# Connecting ClearML with the current process, +# from here on everything is logged automatically task = Task.init(project_name="examples", task_name="scikit-learn joblib example") iris = datasets.load_iris() diff --git a/examples/frameworks/scikit-learn/sklearn_matplotlib_example.py b/examples/frameworks/scikit-learn/sklearn_matplotlib_example.py index de5a3a63..0a305c46 100644 --- a/examples/frameworks/scikit-learn/sklearn_matplotlib_example.py +++ b/examples/frameworks/scikit-learn/sklearn_matplotlib_example.py @@ -124,6 +124,8 @@ def plot_learning_curve(estimator, title, X, y, axes=None, ylim=None, cv=None, n return plt +# Connecting ClearML with the current process, +# from here on everything is logged automatically Task.init('examples', 'scikit-learn matplotlib example') fig, fig_axes = plt.subplots(1, 3, figsize=(30, 10)) diff --git a/examples/frameworks/tensorboardx/pytorch_tensorboardX.py b/examples/frameworks/tensorboardx/pytorch_tensorboardX.py index 498c09fa..bc4fd9e0 100644 --- a/examples/frameworks/tensorboardx/pytorch_tensorboardX.py +++ b/examples/frameworks/tensorboardx/pytorch_tensorboardX.py @@ -100,7 +100,10 @@ def main(): args = parser.parse_args() args.cuda = not args.no_cuda and torch.cuda.is_available() - task = Task.init(project_name='examples', task_name='pytorch with tensorboardX') # noqa: F841 + # Connecting ClearML with the current process, + # from here on everything is logged automatically + task = Task.init(project_name='examples', task_name='pytorch with tensorboardX') + writer = SummaryWriter('runs') writer.add_text('TEXT', 'This is some text', 0) diff --git a/examples/frameworks/tensorflow/legacy/tensorboard_pr_curve.py b/examples/frameworks/tensorflow/legacy/tensorboard_pr_curve.py index 441aba41..70349425 100644 --- a/examples/frameworks/tensorflow/legacy/tensorboard_pr_curve.py +++ b/examples/frameworks/tensorflow/legacy/tensorboard_pr_curve.py @@ -39,6 +39,9 @@ import tensorflow as tf from tensorboard.plugins.pr_curve import summary from clearml import Task + +# Connecting ClearML with the current process, +# from here on everything is logged automatically task = Task.init(project_name='examples', task_name='tensorboard pr_curve') tf.compat.v1.disable_v2_behavior() diff --git a/examples/frameworks/tensorflow/legacy/tensorboard_toy.py b/examples/frameworks/tensorflow/legacy/tensorboard_toy.py index f3b7832d..b53cce77 100644 --- a/examples/frameworks/tensorflow/legacy/tensorboard_toy.py +++ b/examples/frameworks/tensorflow/legacy/tensorboard_toy.py @@ -8,9 +8,12 @@ import numpy as np from PIL import Image from clearml import Task -task = Task.init(project_name='examples', task_name='tensorboard toy example') +# Connecting ClearML with the current process, +# from here on everything is logged automatically +task = Task.init(project_name='examples', task_name='tensorboard toy example') + k = tf.placeholder(tf.float32) # Make a normal distribution, with a shifting mean diff --git a/examples/frameworks/tensorflow/legacy/tensorflow_eager.py b/examples/frameworks/tensorflow/legacy/tensorflow_eager.py index 8f198991..d1f01bd1 100644 --- a/examples/frameworks/tensorflow/legacy/tensorflow_eager.py +++ b/examples/frameworks/tensorflow/legacy/tensorflow_eager.py @@ -32,11 +32,13 @@ from tensorflow.examples.tutorials.mnist import input_data from clearml import Task + tf.compat.v1.enable_eager_execution() +# Connecting ClearML with the current process, +# from here on everything is logged automatically task = Task.init(project_name='examples', task_name='Tensorflow eager mode') - FLAGS = tf.app.flags.FLAGS tf.app.flags.DEFINE_integer('data_num', 100, """Flag of type integer""") tf.app.flags.DEFINE_string('img_path', './img', """Flag of type string""") diff --git a/examples/frameworks/tensorflow/legacy/tensorflow_mnist_with_summaries.py b/examples/frameworks/tensorflow/legacy/tensorflow_mnist_with_summaries.py index 3e06fd80..5aa0f39c 100644 --- a/examples/frameworks/tensorflow/legacy/tensorflow_mnist_with_summaries.py +++ b/examples/frameworks/tensorflow/legacy/tensorflow_mnist_with_summaries.py @@ -34,6 +34,9 @@ from tensorflow.examples.tutorials.mnist import input_data from clearml import Task FLAGS = None + +# Connecting ClearML with the current process, +# from here on everything is logged automatically task = Task.init(project_name='examples', task_name='Tensorflow mnist with summaries example') diff --git a/examples/frameworks/tensorflow/manual_model_upload.py b/examples/frameworks/tensorflow/manual_model_upload.py index 86f8dbe3..f05508f8 100644 --- a/examples/frameworks/tensorflow/manual_model_upload.py +++ b/examples/frameworks/tensorflow/manual_model_upload.py @@ -6,6 +6,9 @@ import tempfile import tensorflow as tf from clearml import Task + +# Connecting ClearML with the current process, +# from here on everything is logged automatically task = Task.init(project_name='examples', task_name='Model configuration and upload') model = tf.Module() diff --git a/examples/frameworks/tensorflow/tensorboard_pr_curve.py b/examples/frameworks/tensorflow/tensorboard_pr_curve.py index 686a3a06..2ab11646 100644 --- a/examples/frameworks/tensorflow/tensorboard_pr_curve.py +++ b/examples/frameworks/tensorflow/tensorboard_pr_curve.py @@ -39,6 +39,8 @@ from tensorboard.plugins.pr_curve import summary from clearml import Task +# Connecting ClearML with the current process, +# from here on everything is logged automatically task = Task.init(project_name='examples', task_name='tensorboard pr_curve') tf.compat.v1.disable_v2_behavior() diff --git a/examples/frameworks/tensorflow/tensorflow_mnist.py b/examples/frameworks/tensorflow/tensorflow_mnist.py index f663648e..f3bc4cf0 100644 --- a/examples/frameworks/tensorflow/tensorflow_mnist.py +++ b/examples/frameworks/tensorflow/tensorflow_mnist.py @@ -11,8 +11,9 @@ from tensorflow.keras import Model from clearml import Task -task = Task.init(project_name='examples', - task_name='Tensorflow v2 mnist with summaries') +# Connecting ClearML with the current process, +# from here on everything is logged automatically +task = Task.init(project_name='examples', task_name='Tensorflow v2 mnist with summaries') # Load and prepare the MNIST dataset. diff --git a/examples/frameworks/xgboost/xgboost_sample.py b/examples/frameworks/xgboost/xgboost_sample.py index 8157ed6e..4d4f292e 100644 --- a/examples/frameworks/xgboost/xgboost_sample.py +++ b/examples/frameworks/xgboost/xgboost_sample.py @@ -7,7 +7,11 @@ from xgboost import plot_tree from clearml import Task + +# Connecting ClearML with the current process, +# from here on everything is logged automatically task = Task.init(project_name='examples', task_name='XGBoost simple example') + iris = datasets.load_iris() X = iris.data y = iris.target @@ -56,5 +60,6 @@ labels = dtest.get_label() # plot results xgb.plot_importance(model) +plt.show() plot_tree(model) plt.show() diff --git a/examples/optimization/hyper-parameter-optimization/base_template_keras_simple.py b/examples/optimization/hyper-parameter-optimization/base_template_keras_simple.py index cf97f1ff..e6e46f41 100644 --- a/examples/optimization/hyper-parameter-optimization/base_template_keras_simple.py +++ b/examples/optimization/hyper-parameter-optimization/base_template_keras_simple.py @@ -20,7 +20,8 @@ from tensorflow.keras.optimizers import RMSprop from clearml import Task, Logger -# Connecting ClearML +# Connecting ClearML with the current process, +# from here on everything is logged automatically task = Task.init(project_name='examples', task_name='Keras HP optimization base') diff --git a/examples/optimization/hyper-parameter-optimization/hyper_parameter_optimizer.py b/examples/optimization/hyper-parameter-optimization/hyper_parameter_optimizer.py index 45a1eafe..84ae1acd 100644 --- a/examples/optimization/hyper-parameter-optimization/hyper_parameter_optimizer.py +++ b/examples/optimization/hyper-parameter-optimization/hyper_parameter_optimizer.py @@ -40,7 +40,8 @@ def job_complete_callback( print('WOOT WOOT we broke the record! Objective reached {}'.format(objective_value)) -# Connecting ClearML +# Connecting ClearML with the current process, +# from here on everything is logged automatically task = Task.init(project_name='Hyper-Parameter Optimization', task_name='Automatic Hyper-Parameter Optimization', task_type=Task.TaskTypes.optimizer, @@ -73,11 +74,10 @@ an_optimizer = HyperParameterOptimizer( UniformIntegerParameterRange('General/layer_2', min_value=128, max_value=512, step_size=128), DiscreteParameterRange('General/batch_size', values=[96, 128, 160]), DiscreteParameterRange('General/epochs', values=[30]), - DiscreteParameterRange('General/optimizer', values=['adam', 'sgd']), ], # this is the objective metric we want to maximize/minimize - objective_metric_title='accuracy', - objective_metric_series='accuracy', + objective_metric_title='epoch_accuracy', + objective_metric_series='epoch_accuracy', # now we decide if we want to maximize it or minimize it (accuracy we maximize) objective_metric_sign='max', # let us limit the number of concurrent experiments, diff --git a/examples/pipeline/pipeline_controller.py b/examples/pipeline/pipeline_controller.py index 3211c6c6..47883b91 100644 --- a/examples/pipeline/pipeline_controller.py +++ b/examples/pipeline/pipeline_controller.py @@ -2,6 +2,8 @@ from clearml import Task from clearml.automation.controller import PipelineController +# Connecting ClearML with the current process, +# from here on everything is logged automatically task = Task.init(project_name='examples', task_name='pipeline demo', task_type=Task.TaskTypes.controller, reuse_last_task_id=False) diff --git a/examples/pipeline/step2_data_processing.py b/examples/pipeline/step2_data_processing.py index dca47520..fcd0bbd5 100644 --- a/examples/pipeline/step2_data_processing.py +++ b/examples/pipeline/step2_data_processing.py @@ -3,7 +3,8 @@ from clearml import Task, StorageManager from sklearn.model_selection import train_test_split -# Connecting ClearML +# Connecting ClearML with the current process, +# from here on everything is logged automatically task = Task.init(project_name="examples", task_name="pipeline step 2 process dataset") # program arguments diff --git a/examples/pipeline/step3_train_model.py b/examples/pipeline/step3_train_model.py index a705588c..2aff66f0 100644 --- a/examples/pipeline/step3_train_model.py +++ b/examples/pipeline/step3_train_model.py @@ -5,7 +5,9 @@ from sklearn.linear_model import LogisticRegression from clearml import Task -# Connecting ClearML + +# Connecting ClearML with the current process, +# from here on everything is logged automatically task = Task.init(project_name="examples", task_name="pipeline step 3 train model") # Arguments diff --git a/examples/reporting/3d_plots_reporting.py b/examples/reporting/3d_plots_reporting.py index 9a41f249..05bdd5d2 100644 --- a/examples/reporting/3d_plots_reporting.py +++ b/examples/reporting/3d_plots_reporting.py @@ -39,7 +39,8 @@ def report_plots(logger, iteration=0): def main(): - # Create the experiment Task + # Connecting ClearML with the current process, + # from here on everything is logged automatically task = Task.init(project_name="examples", task_name="3D plot reporting") print('reporting 3D plot graphs') diff --git a/examples/reporting/artifacts.py b/examples/reporting/artifacts.py index 25fe22c2..b23ff07c 100644 --- a/examples/reporting/artifacts.py +++ b/examples/reporting/artifacts.py @@ -6,7 +6,10 @@ import numpy as np from PIL import Image from clearml import Task -task = Task.init('examples', 'artifacts example') + +# Connecting ClearML with the current process, +# from here on everything is logged automatically +task = Task.init(project_name='examples', task_name='artifacts example') df = pd.DataFrame( { diff --git a/examples/reporting/html_reporting.py b/examples/reporting/html_reporting.py index b6abd0bc..403dc52f 100644 --- a/examples/reporting/html_reporting.py +++ b/examples/reporting/html_reporting.py @@ -217,7 +217,8 @@ def report_html_image(logger, iteration=0): def main(): - # Create the experiment Task + # Connecting ClearML with the current process, + # from here on everything is logged automatically task = Task.init(project_name="examples", task_name="html samples reporting") print('reporting html files into debug samples section') diff --git a/examples/reporting/hyper_parameters.py b/examples/reporting/hyper_parameters.py index 9e05452e..b3802781 100644 --- a/examples/reporting/hyper_parameters.py +++ b/examples/reporting/hyper_parameters.py @@ -19,6 +19,8 @@ FLAGS = flags.FLAGS flags.DEFINE_string('echo', None, 'Text to echo.') flags.DEFINE_string('another_str', 'My string', 'A string', module_name='test') +# Connecting ClearML with the current process, +# from here on everything is logged automatically task = Task.init(project_name='examples', task_name='hyper-parameters example') flags.DEFINE_integer('echo3', 3, 'Text to echo.') @@ -32,8 +34,7 @@ parameters = { 'float': 2.2, 'string': 'my string', } -from clearml import Task -parameters = Task.current_task().connect(parameters, name='more_stuff_deep_inside_code') +parameters = task.connect(parameters) # adding new parameter after connect (will be logged as well) parameters['new_param'] = 'this is new' diff --git a/examples/reporting/image_reporting.py b/examples/reporting/image_reporting.py index 19cc6967..11671c1b 100644 --- a/examples/reporting/image_reporting.py +++ b/examples/reporting/image_reporting.py @@ -43,7 +43,8 @@ def report_debug_images(logger, iteration=0): def main(): - # Create the experiment Task + # Connecting ClearML with the current process, + # from here on everything is logged automatically task = Task.init(project_name="examples", task_name="image reporting") print('reporting a few debug images') diff --git a/examples/reporting/matplotlib_manual_reporting.py b/examples/reporting/matplotlib_manual_reporting.py index 1d6b8e63..dea54931 100644 --- a/examples/reporting/matplotlib_manual_reporting.py +++ b/examples/reporting/matplotlib_manual_reporting.py @@ -4,6 +4,8 @@ import numpy as np import matplotlib.pyplot as plt from clearml import Task +# Connecting ClearML with the current process, +# from here on everything is logged automatically # Create a new task, disable automatic matplotlib connect task = Task.init( project_name='examples', diff --git a/examples/reporting/media_reporting.py b/examples/reporting/media_reporting.py index 7fb9007a..122feb91 100644 --- a/examples/reporting/media_reporting.py +++ b/examples/reporting/media_reporting.py @@ -4,6 +4,8 @@ import os from clearml import Task, Logger +# Connecting ClearML with the current process, +# from here on everything is logged automatically task = Task.init(project_name="examples", task_name="audio and video reporting") print('reporting audio and video samples to the debug samples section') diff --git a/examples/reporting/model_config.py b/examples/reporting/model_config.py index bcfd3170..4cc8f1bf 100644 --- a/examples/reporting/model_config.py +++ b/examples/reporting/model_config.py @@ -5,6 +5,8 @@ import os from clearml import Task, OutputModel +# Connecting ClearML with the current process, +# from here on everything is logged automatically task = Task.init(project_name='examples', task_name='Model configuration example') # Connect a local configuration file diff --git a/examples/reporting/pandas_reporting.py b/examples/reporting/pandas_reporting.py index 1792b254..89391c83 100644 --- a/examples/reporting/pandas_reporting.py +++ b/examples/reporting/pandas_reporting.py @@ -43,7 +43,8 @@ def report_table(logger, iteration=0): def main(): - # Create the experiment Task + # Connecting ClearML with the current process, + # from here on everything is logged automatically task = Task.init(project_name="examples", task_name="table reporting") print('reporting pandas tables and python lists as tables into the plots section') diff --git a/examples/reporting/plotly_reporting.py b/examples/reporting/plotly_reporting.py index dcbabc9b..56eb7ff2 100644 --- a/examples/reporting/plotly_reporting.py +++ b/examples/reporting/plotly_reporting.py @@ -4,6 +4,8 @@ from clearml import Task import plotly.express as px +# Connecting ClearML with the current process, +# from here on everything is logged automatically task = Task.init('examples', 'plotly reporting') print('reporting plotly figures') diff --git a/examples/reporting/scalar_reporting.py b/examples/reporting/scalar_reporting.py index adda0433..6dd68dd8 100644 --- a/examples/reporting/scalar_reporting.py +++ b/examples/reporting/scalar_reporting.py @@ -21,7 +21,8 @@ def report_scalars(logger): def main(): - # Create the experiment Task + # Connecting ClearML with the current process, + # from here on everything is logged automatically task = Task.init(project_name="examples", task_name="scalar reporting") print('reporting scalar graphs') diff --git a/examples/reporting/scatter_hist_confusion_mat_reporting.py b/examples/reporting/scatter_hist_confusion_mat_reporting.py index 72b79318..970747ee 100644 --- a/examples/reporting/scatter_hist_confusion_mat_reporting.py +++ b/examples/reporting/scatter_hist_confusion_mat_reporting.py @@ -109,7 +109,8 @@ def report_plots(logger, iteration=0): def main(): - # Create the experiment Task + # Connecting ClearML with the current process, + # from here on everything is logged automatically task = Task.init(project_name="examples", task_name="2D plots reporting") print('reporting some graphs') @@ -117,7 +118,6 @@ def main(): # Get the task logger, # You can also call Task.current_task().get_logger() from anywhere in your code. logger = task.get_logger() - #logger.report_scatter2d() # report graphs report_plots(logger) diff --git a/examples/reporting/text_reporting.py b/examples/reporting/text_reporting.py index 99209ea9..bc77c8c2 100644 --- a/examples/reporting/text_reporting.py +++ b/examples/reporting/text_reporting.py @@ -58,7 +58,8 @@ Vestibulum dictum ipsum at viverra ultrices. Aliquam sed ante massa. Quisque con def main(): - # Create the experiment Task + # Connecting ClearML with the current process, + # from here on everything is logged automatically task = Task.init(project_name="examples", task_name="text reporting") print("reporting text logs") diff --git a/examples/services/aws-autoscaler/aws_autoscaler.py b/examples/services/aws-autoscaler/aws_autoscaler.py index e9335bab..c5af7549 100644 --- a/examples/services/aws-autoscaler/aws_autoscaler.py +++ b/examples/services/aws-autoscaler/aws_autoscaler.py @@ -76,6 +76,8 @@ def main(): ) return + # Connecting ClearML with the current process, + # from here on everything is logged automatically task = Task.init(project_name="DevOps", task_name="AWS Auto-Scaler", task_type=Task.TaskTypes.service) task.connect(hyper_params) task.connect_configuration(configurations) diff --git a/examples/services/cleanup/cleanup_service.py b/examples/services/cleanup/cleanup_service.py index 7939cebe..a804a99a 100644 --- a/examples/services/cleanup/cleanup_service.py +++ b/examples/services/cleanup/cleanup_service.py @@ -24,7 +24,8 @@ from clearml.backend_api.session.client import APIClient from clearml import Task -# Connecting ClearML +# Connecting ClearML with the current process, +# from here on everything is logged automatically task = Task.init( project_name="DevOps", task_name="Cleanup Service", diff --git a/examples/services/jupyter-service/execute_jupyter_notebook_server.py b/examples/services/jupyter-service/execute_jupyter_notebook_server.py index 0c30ad6b..87c26042 100644 --- a/examples/services/jupyter-service/execute_jupyter_notebook_server.py +++ b/examples/services/jupyter-service/execute_jupyter_notebook_server.py @@ -12,9 +12,13 @@ import jupyter # noqa from clearml import Task -# initialize ClearML +# Connecting ClearML with the current process, +# from here on everything is logged automatically task = Task.init( - project_name="DevOps", task_name="Allocate Jupyter Notebook Instance", task_type=Task.TaskTypes.service) + project_name="DevOps", + task_name="Allocate Jupyter Notebook Instance", + task_type=Task.TaskTypes.service +) # get rid of all the runtime ClearML preserve = ( diff --git a/examples/services/monitoring/slack_alerts.py b/examples/services/monitoring/slack_alerts.py index 5ea5a89c..471b0fe8 100644 --- a/examples/services/monitoring/slack_alerts.py +++ b/examples/services/monitoring/slack_alerts.py @@ -192,6 +192,8 @@ def main(): slack_monitor.status_alerts += ["completed"] # start the monitoring Task + # Connecting ClearML with the current process, + # from here on everything is logged automatically task = Task.init(project_name='Monitoring', task_name='Slack Alerts', task_type=Task.TaskTypes.monitor) if not args.local: task.execute_remotely(queue_name=args.service_queue)