clearml initial version 0.17.0

This commit is contained in:
allegroai
2020-12-22 23:25:37 +02:00
parent a460df1e68
commit d327f2e2b9
145 changed files with 3136 additions and 794 deletions

View File

@@ -3,7 +3,7 @@ import numpy as np
import tensorflow as tf
from tensorflow import keras
from trains import Task
from clearml import Task
task = Task.init(project_name="autokeras", task_name="autokeras imdb example with scalars")

View File

@@ -1,5 +1,5 @@
# Plese read this https://github.com/keras-team/autokeras#installation before doing changes
autokeras
tensorflow>=2.3.0
trains
clearml
git+https://github.com/keras-team/keras-tuner.git@1.0.2rc2

View File

@@ -1,10 +1,10 @@
# TRAINS - Fastai with Tensorboard example code, automatic logging the model and Tensorboard outputs
# ClearML - Fastai with Tensorboard example code, automatic logging the model and Tensorboard outputs
#
from fastai.callbacks.tensorboard import LearnerTensorboardWriter
from fastai.vision import * # Quick access to computer vision functionality
from trains import Task
from clearml import Task
task = Task.init(project_name="example", task_name="fastai with tensorboard callback")

View File

@@ -1,4 +1,4 @@
fastai
tensorboard
tensorboardX
trains
clearml

View File

@@ -15,12 +15,12 @@ from ignite.utils import setup_logger
from torch.utils.tensorboard import SummaryWriter
from tqdm import tqdm
from trains import Task, StorageManager
from clearml import Task, StorageManager
# Trains Initializations
# ClearML Initializations
task = Task.init(project_name='Image Example', task_name='image classification CIFAR10')
params = {'number_of_epochs': 20, 'batch_size': 64, 'dropout': 0.25, 'base_lr': 0.001, 'momentum': 0.9, 'loss_report': 100}
params = task.connect(params) # enabling configuration override by trains
params = task.connect(params) # enabling configuration override by clearml
print(params) # printing actual configuration (after override in remote mode)
manager = StorageManager()

View File

@@ -1,4 +1,4 @@
# TRAINS - Keras with Tensorboard example code, automatic logging model and Tensorboard outputs
# ClearML - Keras with Tensorboard example code, automatic logging model and Tensorboard outputs
#
# Train a simple deep NN on the MNIST dataset.
# Gets to 98.40% test accuracy after 20 epochs
@@ -19,7 +19,7 @@ from tensorflow.keras.layers import Activation, Dense
from tensorflow.keras.models import Sequential
from tensorflow.keras.optimizers import RMSprop
from trains import Task
from clearml import Task
class TensorBoardImage(TensorBoard):
@@ -89,7 +89,7 @@ model.compile(loss='categorical_crossentropy',
optimizer=RMSprop(),
metrics=['accuracy'])
# Connecting TRAINS
# Connecting ClearML
task = Task.init(project_name='examples', task_name='Keras with TensorBoard example')
# To set your own configuration:

View File

@@ -1,4 +1,4 @@
# TRAINS - Keras with Tensorboard example code, automatic logging model and Tensorboard outputs
# ClearML - Keras with Tensorboard example code, automatic logging model and Tensorboard outputs
#
# Train a simple deep NN on the MNIST dataset.
# Gets to 98.40% test accuracy after 20 epochs
@@ -18,7 +18,7 @@ from keras.layers.core import Dense, Activation
from keras.optimizers import RMSprop
from keras.utils import np_utils
import tensorflow as tf
from trains import Task
from clearml import Task
class TensorBoardImage(TensorBoard):
@@ -88,7 +88,7 @@ model.compile(loss='categorical_crossentropy',
optimizer=RMSprop(),
metrics=['accuracy'])
# Connecting TRAINS
# Connecting ClearML
task = Task.init(project_name='examples', task_name='Keras with TensorBoard example')
task.connect_configuration({'test': 1337, 'nested': {'key': 'value', 'number': 1}})

View File

@@ -1,2 +1,2 @@
trains
clearml
Keras>=2.2.4

View File

@@ -1,11 +1,11 @@
# TRAINS - Example of manual model configuration and uploading
# ClearML - Example of manual model configuration and uploading
#
import os
from tempfile import gettempdir
from keras import Input, layers, Model
from trains import Task
from clearml import Task
task = Task.init(project_name='examples', task_name='Model configuration and upload')

View File

@@ -1,3 +1,3 @@
Keras
tensorflow>=2.0
trains
clearml

View File

@@ -3,9 +3,9 @@
import kerastuner as kt
import tensorflow as tf
import tensorflow_datasets as tfds
from trains.external.kerastuner import TrainsTunerLogger
from clearml.external.kerastuner import TrainsTunerLogger
from trains import Task
from clearml import Task
physical_devices = tf.config.list_physical_devices('GPU')
if physical_devices:

View File

@@ -1,4 +1,4 @@
keras-tuner
tensorflow>=2.0
tensorflow-datasets
trains
clearml

View File

@@ -1,4 +1,4 @@
lightgbm
scikit-learn
pandas
trains
clearml

View File

@@ -1,10 +1,10 @@
# TRAINS - Example of LightGBM integration
# ClearML - Example of LightGBM integration
#
import lightgbm as lgb
import pandas as pd
from sklearn.metrics import mean_squared_error
from trains import Task
from clearml import Task
task = Task.init(project_name="examples", task_name="LIGHTgbm")

View File

@@ -1,9 +1,9 @@
# TRAINS - Example of Matplotlib and Seaborn integration and reporting
# ClearML - Example of Matplotlib and Seaborn integration and reporting
#
import numpy as np
import matplotlib.pyplot as plt
import seaborn as sns
from trains import Task
from clearml import Task
task = Task.init(project_name='examples', task_name='Matplotlib example')

View File

@@ -1,4 +1,4 @@
matplotlib >= 3.1.1 ; python_version >= '3.6'
matplotlib >= 2.2.4 ; python_version < '3.6'
seaborn
trains
clearml

View File

@@ -1,10 +1,10 @@
# TRAINS - Example of manual model configuration and uploading
# ClearML - Example of manual model configuration and uploading
#
import os
from tempfile import gettempdir
import torch
from trains import Task
from clearml import Task
task = Task.init(project_name='examples', task_name='Model configuration and upload')

View File

@@ -1,4 +1,4 @@
# TRAINS - example of TRAINS torch distributed support
# ClearML - example of ClearML torch distributed support
# notice all nodes will be reporting to the master Task (experiment)
import os
@@ -15,7 +15,7 @@ import torch.nn.functional as F
from torch import optim
from torchvision import datasets, transforms
from trains import Task
from clearml import Task
local_dataset_path = './MNIST_data'
@@ -150,7 +150,7 @@ if __name__ == "__main__":
# We have to initialize the task in the master process,
# it will make sure that any sub-process calling Task.init will get the master task object
# notice that we exclude the `rank` argument, so we can launch multiple sub-processes with trains-agent
# notice that we exclude the `rank` argument, so we can launch multiple sub-processes with clearml-agent
# otherwise, the `rank` will always be set to the original value.
task = Task.init("examples", "test torch distributed", auto_connect_arg_parser={'rank': False})

View File

@@ -1,4 +1,4 @@
# TRAINS - Example of Pytorch and matplotlib integration and reporting
# ClearML - Example of Pytorch and matplotlib integration and reporting
#
"""
Neural Transfer Using PyTorch
@@ -60,7 +60,7 @@ import torchvision.transforms as transforms
import torchvision.models as models
import copy
from trains import Task
from clearml import Task
task = Task.init(project_name='examples', task_name='pytorch with matplotlib example', task_type=Task.TaskTypes.testing)

View File

@@ -1,4 +1,4 @@
# TRAINS - Example of Pytorch mnist training integration
# ClearML - Example of Pytorch mnist training integration
#
from __future__ import print_function
import argparse
@@ -11,7 +11,7 @@ import torch.nn.functional as F
import torch.optim as optim
from torchvision import datasets, transforms
from trains import Task, Logger
from clearml import Task, Logger
class Net(nn.Module):

View File

@@ -1,4 +1,4 @@
# TRAINS - Example of pytorch with tensorboard>=v1.14
# ClearML - Example of pytorch with tensorboard>=v1.14
#
from __future__ import print_function
@@ -14,7 +14,7 @@ from torchvision import datasets, transforms
from torch.autograd import Variable
from torch.utils.tensorboard import SummaryWriter
from trains import Task
from clearml import Task
class Net(nn.Module):
@@ -99,7 +99,7 @@ def main():
parser.add_argument('--log-interval', type=int, default=10, metavar='N',
help='how many batches to wait before logging training status')
args = parser.parse_args()
task = Task.init(project_name='examples', task_name='pytorch with tensorboard') # noqa: F841
Task.init(project_name='examples', task_name='pytorch with tensorboard')
writer = SummaryWriter('runs')
writer.add_text('TEXT', 'This is some text', 0)
args.cuda = not args.no_cuda and torch.cuda.is_available()

View File

@@ -3,4 +3,4 @@ tensorboardX
tensorboard>=1.14.0
torch>=1.1.0
torchvision>=0.3.0
trains
clearml

View File

@@ -5,7 +5,7 @@ import numpy as np
from PIL import Image
from torch.utils.tensorboard import SummaryWriter
from trains import Task
from clearml import Task
task = Task.init(project_name='examples', task_name='pytorch tensorboard toy example')

View File

@@ -2,4 +2,4 @@ joblib>=0.13.2
matplotlib >= 3.1.1 ; python_version >= '3.6'
matplotlib >= 2.2.4 ; python_version < '3.6'
scikit-learn
trains
clearml

View File

@@ -10,7 +10,7 @@ import numpy as np
import matplotlib.pyplot as plt
from trains import Task
from clearml import Task
task = Task.init(project_name="examples", task_name="scikit-learn joblib example")

View File

@@ -6,7 +6,7 @@ from sklearn.model_selection import learning_curve
from sklearn.naive_bayes import GaussianNB
from sklearn.svm import SVC
from trains import Task
from clearml import Task
def plot_learning_curve(estimator, title, X, y, axes=None, ylim=None, cv=None, n_jobs=None,

View File

@@ -1,4 +1,4 @@
# TRAINS - Example of pytorch with tensorboardX
# ClearML - Example of pytorch with tensorboardX
#
from __future__ import print_function
@@ -14,7 +14,7 @@ from tensorboardX import SummaryWriter
from torch.autograd import Variable
from torchvision import datasets, transforms
from trains import Task
from clearml import Task
class Net(nn.Module):

View File

@@ -1,4 +1,4 @@
tensorboardX>=1.8
torch>=1.1.0
torchvision>=0.3.0
trains
clearml

View File

@@ -1,4 +1,4 @@
# TRAINS - Example of new tensorboard pr_curves model
# ClearML - Example of new tensorboard pr_curves model
#
# Copyright 2017 The TensorFlow Authors. All Rights Reserved.
#
@@ -37,7 +37,7 @@ from absl import flags
from six.moves import xrange # pylint: disable=redefined-builtin
import tensorflow as tf
from tensorboard.plugins.pr_curve import summary
from trains import Task
from clearml import Task
task = Task.init(project_name='examples', task_name='tensorboard pr_curve')

View File

@@ -1,4 +1,4 @@
# TRAINS - Example of tensorboard with tensorflow (without any actual training)
# ClearML - Example of tensorboard with tensorflow (without any actual training)
#
import os
from tempfile import gettempdir
@@ -7,7 +7,7 @@ import tensorflow as tf
import numpy as np
from PIL import Image
from trains import Task
from clearml import Task
task = Task.init(project_name='examples', task_name='tensorboard toy example')

View File

@@ -1,4 +1,4 @@
# TRAINS - Example of tensorflow eager mode, model logging and tensorboard
# ClearML - Example of tensorflow eager mode, model logging and tensorboard
#
# Copyright 2017 The TensorFlow Authors. All Rights Reserved.
#
@@ -30,7 +30,7 @@ from tempfile import gettempdir
import tensorflow as tf
from tensorflow.examples.tutorials.mnist import input_data
from trains import Task
from clearml import Task
tf.compat.v1.enable_eager_execution()

View File

@@ -31,7 +31,7 @@ import tempfile
import tensorflow as tf
from tensorflow.examples.tutorials.mnist import input_data
from trains import Task
from clearml import Task
FLAGS = None
task = Task.init(project_name='examples', task_name='Tensorflow mnist with summaries example')

View File

@@ -1,10 +1,10 @@
# TRAINS - Example of manual model configuration and uploading
# ClearML - Example of manual model configuration and uploading
#
import os
import tempfile
import tensorflow as tf
from trains import Task
from clearml import Task
task = Task.init(project_name='examples', task_name='Model configuration and upload')

View File

@@ -1,3 +1,3 @@
tensorboard>=2.0
tensorflow>=2.0
trains
clearml

View File

@@ -36,7 +36,7 @@ from six.moves import xrange # pylint: disable=redefined-builtin
import tensorflow as tf
from tensorboard.plugins.pr_curve import summary
from trains import Task
from clearml import Task
task = Task.init(project_name='examples', task_name='tensorboard pr_curve')

View File

@@ -1,4 +1,4 @@
# TRAINS - Example of tensorboard with tensorflow (without any actual training)
# ClearML - Example of tensorboard with tensorflow (without any actual training)
#
import os
import tensorflow as tf
@@ -6,7 +6,7 @@ import numpy as np
from tempfile import gettempdir
from PIL import Image
from trains import Task
from clearml import Task
def generate_summary(k, step):

View File

@@ -8,7 +8,7 @@ import tensorflow as tf
from tensorflow.keras.layers import Dense, Flatten, Conv2D
from tensorflow.keras import Model
from trains import Task
from clearml import Task
task = Task.init(project_name='examples',

View File

@@ -1,7 +1,7 @@
matplotlib >= 3.1.1 ; python_version >= '3.6'
matplotlib >= 2.2.4 ; python_version < '3.6'
sklearn
trains
clearml
xgboost>=0.90 ; python_version >= '3'
xgboost>=0.82 ; python_version < '3'
# sudo apt-get install graphviz

View File

@@ -5,7 +5,7 @@ from sklearn.metrics import accuracy_score
from sklearn.model_selection import train_test_split
from xgboost import plot_tree
from trains import Task
from clearml import Task
task = Task.init(project_name='examples', task_name='XGBoost simple example')
iris = datasets.load_iris()