ClearML/clearml
1
0
Fork 0
mirror of https://github.com/clearml/clearml synced 2025-06-26 18:16:07 +00:00
Code Issues Actions Packages Projects Releases Wiki Activity

Update examples to ClearML

Browse Source
This commit is contained in:
allegroai
2020-12-30 16:53:19 +02:00
parent 0bb590b5da
commit 4c8a24f1ff
17 changed files with 86 additions and 87 deletions
Download Patch File Download Diff File Expand all files Collapse all files

12
examples/frameworks/pytorch/notebooks/audio/audio_classifier_UrbanSound8K.ipynb
View File

@@ -15,7 +15,7 @@
"! pip install -U torchaudio==0.5.1\n",
"! pip install -U torchvision==0.6.1\n",
"! pip install -U matplotlib==3.2.1\n",
"! pip install -U trains>=0.16.1\n",
"! pip install -U clearml>=0.16.1\n",
"! pip install -U pandas==1.0.4\n",
"! pip install -U numpy==1.18.4\n",
"! pip install -U tensorboard==2.2.1"
@@ -50,8 +50,8 @@
"from torchvision.transforms import ToTensor\n",
"from torchvision import models\n",
"\n",
"from trains import Task\n",
"from trains.storage import StorageManager\n",
"from clearml import Task\n",
"from clearml.storage import StorageManager\n",
"\n",
"%matplotlib inline"
]
@@ -65,7 +65,7 @@
"task = Task.init(project_name='Audio Example', task_name='audio classification UrbanSound8K')\n",
"configuration_dict = {'number_of_epochs': 3, 'batch_size': 8, 'dropout': 0.3, 'base_lr': 0.005, \n",
" 'number_of_mel_filters': 64, 'resample_freq': 22050}\n",
"configuration_dict = task.connect(configuration_dict) # enabling configuration override by trains\n",
"configuration_dict = task.connect(configuration_dict) # enabling configuration override by clearml\n",
"print(configuration_dict) # printing actual configuration (after override in remote mode)"
]
},
@@ -81,8 +81,8 @@
"outputs": [],
"source": [
"# Download UrbanSound8K dataset (https://urbansounddataset.weebly.com/urbansound8k.html)\n",
"# For simplicity we will use here a subset of that dataset using trains StorageManager\n",
"path_to_UrbanSound8K = StorageManager.get_local_copy(\"https://allegro-datasets.s3.amazonaws.com/trains/UrbanSound8K.zip\", \n",
"# For simplicity we will use here a subset of that dataset using clearml StorageManager\n",
"path_to_UrbanSound8K = StorageManager.get_local_copy(\"https://allegro-datasets.s3.amazonaws.com/clearml/UrbanSound8K.zip\", \n",
" extract_archive=True)\n",
"path_to_UrbanSound8K_csv = Path(path_to_UrbanSound8K) / 'UrbanSound8K' / 'metadata' / 'UrbanSound8K.csv'\n",
"path_to_UrbanSound8K_audio = Path(path_to_UrbanSound8K) / 'UrbanSound8K' / 'audio'"

6
examples/frameworks/pytorch/notebooks/audio/audio_preprocessing_example.ipynb
View File

@@ -12,7 +12,7 @@
"! pip install -U torch==1.5.1\n",
"! pip install -U torchaudio==0.5.1\n",
"! pip install -U matplotlib==3.2.1\n",
"! pip install -U trains>=0.16.1\n",
"! pip install -U clearml>=0.16.1\n",
"! pip install -U tensorboard==2.2.1"
]
},
@@ -28,7 +28,7 @@
"from torch.utils.tensorboard import SummaryWriter\n",
"import matplotlib.pyplot as plt\n",
"\n",
"from trains import Task\n",
"from clearml import Task\n",
"\n",
"%matplotlib inline"
]
@@ -41,7 +41,7 @@
"source": [
"task = Task.init(project_name='Audio Example', task_name='data pre-processing')\n",
"configuration_dict = {'number_of_samples': 3}\n",
"configuration_dict = task.connect(configuration_dict) # enabling configuration override by trains\n",
"configuration_dict = task.connect(configuration_dict) # enabling configuration override by clearml\n",
"print(configuration_dict) # printing actual configuration (after override in remote mode)"
]
},

16
examples/frameworks/pytorch/notebooks/image/hyperparameter_search.ipynb
View File

@@ -7,12 +7,12 @@
"outputs": [],
"source": [
"# execute this in command line on all machines to be used as workers before initiating the hyperparamer search \n",
"# ! pip install -U trains-agent==0.15.0\n",
"# ! trains-agent daemon --queue default\n",
"# ! pip install -U clearml-agent==0.15.0\n",
"# ! clearml-agent daemon --queue default\n",
"\n",
"# pip install with locked versions\n",
"! pip install -U pandas==1.0.3\n",
"! pip install -U trains>=0.16.2\n",
"! pip install -U clearml>=0.16.2\n",
"! pip install -U optuna==2.0.0"
]
},
@@ -22,11 +22,11 @@
"metadata": {},
"outputs": [],
"source": [
"from trains.automation import UniformParameterRange, UniformIntegerParameterRange\n",
"from trains.automation import HyperParameterOptimizer\n",
"from trains.automation.optuna import OptimizerOptuna\n",
"from clearml.automation import UniformParameterRange, UniformIntegerParameterRange\n",
"from clearml.automation import HyperParameterOptimizer\n",
"from clearml.automation.optuna import OptimizerOptuna\n",
"\n",
"from trains import Task"
"from clearml import Task"
]
},
{
@@ -72,7 +72,7 @@
" objective_metric_series='total',\n",
" objective_metric_sign='max', # maximize or minimize the objective metric\n",
"\n",
" # setting optimizer - trains supports GridSearch, RandomSearch, OptimizerBOHB and OptimizerOptuna\n",
" # setting optimizer - clearml supports GridSearch, RandomSearch, OptimizerBOHB and OptimizerOptuna\n",
" optimizer_class=OptimizerOptuna,\n",
" \n",
" # Configuring optimization parameters\n",

6
examples/frameworks/pytorch/notebooks/image/image_classification_CIFAR10.ipynb
View File

@@ -15,7 +15,7 @@
"! pip install -U torch==1.5.1\n",
"! pip install -U torchvision==0.6.1\n",
"! pip install -U numpy==1.18.4\n",
"! pip install -U trains>=0.16.1\n",
"! pip install -U clearml>=0.16.1\n",
"! pip install -U tensorboard==2.2.1"
]
},
@@ -36,7 +36,7 @@
"import torchvision.datasets as datasets\n",
"import torchvision.transforms as transforms\n",
"\n",
"from trains import Task"
"from clearml import Task"
]
},
{
@@ -47,7 +47,7 @@
"source": [
"task = Task.init(project_name='Image Example', task_name='image classification CIFAR10')\n",
"configuration_dict = {'number_of_epochs': 3, 'batch_size': 4, 'dropout': 0.25, 'base_lr': 0.001}\n",
"configuration_dict = task.connect(configuration_dict) # enabling configuration override by trains\n",
"configuration_dict = task.connect(configuration_dict) # enabling configuration override by clearml\n",
"print(configuration_dict) # printing actual configuration (after override in remote mode)"
]
},

6
examples/frameworks/pytorch/notebooks/table/download_and_preprocessing.ipynb
View File

@@ -8,7 +8,7 @@
"source": [
"! pip install -U pip\n",
"! pip install -U torch==1.5.1\n",
"! pip install -U trains>=0.15.1\n",
"! pip install -U clearml>=0.15.1\n",
"! pip install -U pandas==1.0.4\n",
"! pip install -U numpy==1.18.4\n",
"! pip install -U pathlib2==2.3.5\n",
@@ -28,7 +28,7 @@
"import torch\n",
"from datetime import datetime\n",
"from pathlib2 import Path\n",
"from trains import Task"
"from clearml import Task"
]
},
{
@@ -40,7 +40,7 @@
"task = Task.init(project_name='Table Example', task_name='tabular preprocessing')\n",
"logger = task.get_logger()\n",
"configuration_dict = {'test_size': 0.1, 'split_random_state': 0}\n",
"configuration_dict = task.connect(configuration_dict) # enabling configuration override by trains\n",
"configuration_dict = task.connect(configuration_dict) # enabling configuration override by clearml\n",
"print(configuration_dict) # printing actual configuration (after override in remote mode)"
]
},

12
examples/frameworks/pytorch/notebooks/table/download_and_split.ipynb
View File

@@ -7,7 +7,7 @@
"outputs": [],
"source": [
"! pip install -U pip\n",
"! pip install -U trains==0.16.2rc0\n",
"! pip install -U clearml==0.16.2rc0\n",
"! pip install -U pandas==1.0.4\n",
"! pip install -U scikit-learn==0.23.1\n",
"! pip install -U pathlib2==2.3.5"
@@ -23,7 +23,7 @@
"from pathlib2 import Path\n",
"from sklearn.model_selection import train_test_split\n",
"\n",
"from trains import Task"
"from clearml import Task"
]
},
{
@@ -35,7 +35,7 @@
"task = Task.init(project_name='Tabular Example', task_name='Download and split tabular dataset')\n",
"logger = task.get_logger()\n",
"configuration_dict = {'test_size': 0.1, 'split_random_state': 0}\n",
"configuration_dict = task.connect(configuration_dict) # enabling configuration override by trains\n",
"configuration_dict = task.connect(configuration_dict) # enabling configuration override by clearml\n",
"print(configuration_dict) # printing actual configuration (after override in remote mode)"
]
},
@@ -54,9 +54,9 @@
"source": [
"# Download the shelter-animal-outcomes dataset (https://www.kaggle.com/c/shelter-animal-outcomes)\n",
"# and save it to your cloud storage or your mounted local storage\n",
"# If the data is on your cloud storage, you can use trains' storage manager to get a local copy of it:\n",
"# from trains.storage import StorageManager\n",
"# path_to_ShelterAnimal = StorageManager.get_local_copy(\"https://allegro-datasets.s3.amazonaws.com/trains/UrbanSound8K.zip\", \n",
"# If the data is on your cloud storage, you can use clearml' storage manager to get a local copy of it:\n",
"# from clearml.storage import StorageManager\n",
"# path_to_ShelterAnimal = StorageManager.get_local_copy(\"https://allegro-datasets.s3.amazonaws.com/clearml/UrbanSound8K.zip\", \n",
"# extract_archive=True)\n",
"path_to_ShelterAnimal = '/home/sam/Datasets/shelter-animal-outcomes'"
]

6
examples/frameworks/pytorch/notebooks/table/pick_best_model.ipynb
View File

@@ -7,7 +7,7 @@
"outputs": [],
"source": [
"! pip install -U pip\n",
"! pip install -U trains==0.16.2rc0"
"! pip install -U clearml==0.16.2rc0"
]
},
{
@@ -16,7 +16,7 @@
"metadata": {},
"outputs": [],
"source": [
"from trains import Task, OutputModel"
"from clearml import Task, OutputModel"
]
},
{
@@ -27,7 +27,7 @@
"source": [
"task = Task.init(project_name='Tabular Example', task_name='pick best model')\n",
"configuration_dict = {'train_tasks_ids': ['c9bff3d15309487a9e5aaa00358ff091', 'c9bff3d15309487a9e5aaa00358ff091']}\n",
"configuration_dict = task.connect(configuration_dict) # enabling configuration override by trains\n",
"configuration_dict = task.connect(configuration_dict) # enabling configuration override by clearml\n",
"print(configuration_dict) # printing actual configuration (after override in remote mode)"
]
},

6
examples/frameworks/pytorch/notebooks/table/preprocessing_and_encoding.ipynb
View File

@@ -7,7 +7,7 @@
"outputs": [],
"source": [
"! pip install -U pip\n",
"! pip install -U trains==0.16.2rc0\n",
"! pip install -U clearml==0.16.2rc0\n",
"! pip install -U pandas==1.0.4\n",
"! pip install -U numpy==1.18.4"
]
@@ -22,7 +22,7 @@
"import numpy as np\n",
"from collections import Counter\n",
"\n",
"from trains import Task"
"from clearml import Task"
]
},
{
@@ -35,7 +35,7 @@
"logger = task.get_logger()\n",
"configuration_dict = {'data_task_id': '39fbf86fc4a341359ac6df4aa70ff91b',\n",
" 'fill_categorical_NA': True, 'fill_numerical_NA': True}\n",
"configuration_dict = task.connect(configuration_dict) # enabling configuration override by trains\n",
"configuration_dict = task.connect(configuration_dict) # enabling configuration override by clearml\n",
"print(configuration_dict) # printing actual configuration (after override in remote mode)"
]
},

6
examples/frameworks/pytorch/notebooks/table/tabular_ml_pipeline.ipynb
View File

@@ -8,7 +8,7 @@
"source": [
"# pip install with locked versions\n",
"! pip install -U pip\n",
"! pip install -U trains==0.16.2rc0"
"! pip install -U clearml==0.16.2rc0"
]
},
{
@@ -17,8 +17,8 @@
"metadata": {},
"outputs": [],
"source": [
"from trains import Task\n",
"from trains.automation.controller import PipelineController"
"from clearml import Task\n",
"from clearml.automation.controller import PipelineController"
]
},
{

6
examples/frameworks/pytorch/notebooks/table/train_tabular_predictor.ipynb
View File

@@ -8,7 +8,7 @@
"source": [
"! pip install -U pip\n",
"! pip install -U torch==1.5.1\n",
"! pip install -U trains==0.16.2rc0\n",
"! pip install -U clearml==0.16.2rc0\n",
"! pip install -U pandas==1.0.4\n",
"! pip install -U numpy==1.18.4\n",
"! pip install -U tensorboard==2.2.1"
@@ -29,7 +29,7 @@
"from torch.utils.data import Dataset\n",
"from torch.utils.tensorboard import SummaryWriter\n",
"\n",
"from trains import Task"
"from clearml import Task"
]
},
{
@@ -42,7 +42,7 @@
"logger = task.get_logger()\n",
"configuration_dict = {'data_task_id': 'b605d76398f941e69fc91b43420151d2', \n",
" 'number_of_epochs': 15, 'batch_size': 100, 'dropout': 0.3, 'base_lr': 0.1}\n",
"configuration_dict = task.connect(configuration_dict) # enabling configuration override by trains\n",
"configuration_dict = task.connect(configuration_dict) # enabling configuration override by clearml\n",
"print(configuration_dict) # printing actual configuration (after override in remote mode)"
]
},

6
examples/frameworks/pytorch/notebooks/text/text_classification_AG_NEWS.ipynb
View File

@@ -10,7 +10,7 @@
"! pip install -U torch==1.5.0\n",
"! pip install -U torchtext==0.6.0\n",
"! pip install -U matplotlib==3.2.1\n",
"! pip install -U trains>=0.15.0\n",
"! pip install -U clearml>=0.15.0\n",
"! pip install -U tensorboard==2.2.1"
]
},
@@ -28,7 +28,7 @@
"from torchtext.datasets import text_classification\n",
"from torch.utils.tensorboard import SummaryWriter\n",
"\n",
"from trains import Task\n",
"from clearml import Task\n",
"\n",
"%matplotlib inline"
]
@@ -41,7 +41,7 @@
"source": [
"task = Task.init(project_name='Text Example', task_name='text classifier')\n",
"configuration_dict = {'number_of_epochs': 6, 'batch_size': 16, 'ngrams': 2, 'base_lr': 1.0}\n",
"configuration_dict = task.connect(configuration_dict) # enabling configuration override by trains\n",
"configuration_dict = task.connect(configuration_dict) # enabling configuration override by clearml\n",
"print(configuration_dict) # printing actual configuration (after override in remote mode)"
]
},