ClearML/clearml
1
0
Fork 0
mirror of https://github.com/clearml/clearml synced 2025-01-31 09:07:00 +00:00
Code Issues Actions Packages Projects Releases Wiki Activity

Edit pipeline example (#494)

Browse Source
This commit is contained in:
pollfly 2021-11-21 12:30:05 +02:00 committed by GitHub
parent 3d4b9c7127
commit 8f84c42f5d
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
1 changed files with 53 additions and 30 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

83
examples/frameworks/pytorch/notebooks/table/tabular_ml_pipeline.ipynb
View File

@ -27,35 +27,58 @@
"metadata": {},
"outputs": [],
"source": [
"task = Task.init(project_name='Tabular Example', task_name='tabular training pipeline', task_type=Task.TaskTypes.controller)"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": [
"pipe = PipelineController(default_execution_queue='dan_queue', add_pipeline_tags=True)\n",
"pipe.add_step(name='preprocessing_1', base_task_project='Tabular Example', base_task_name='tabular preprocessing',\n",
" parameter_override={'General/data_task_id': '39fbf86fc4a341359ac6df4aa70ff91b',\n",
" 'General/fill_categorical_NA': 'True',\n",
" 'General/fill_numerical_NA': 'True'})\n",
"pipe.add_step(name='preprocessing_2', base_task_project='Tabular Example', base_task_name='tabular preprocessing',\n",
" parameter_override={'General/data_task_id': '39fbf86fc4a341359ac6df4aa70ff91b',\n",
" 'General/fill_categorical_NA': 'False',\n",
" 'General/fill_numerical_NA': 'True'})\n",
" \n",
"pipe.add_step(name='train_1', parents=['preprocessing_1'],\n",
" base_task_project='Tabular Example', base_task_name='tabular prediction',\n",
" parameter_override={'General/data_task_id': '${preprocessing_1.id}'})\n",
"pipe.add_step(name='train_2', parents=['preprocessing_2'],\n",
" base_task_project='Tabular Example', base_task_name='tabular prediction',\n",
" parameter_override={'General/data_task_id': '${preprocessing_2.id}'})\n",
" \n",
"pipe.add_step(name='pick_best', parents=['train_1', 'train_2'],\n",
" base_task_project='Tabular Example', base_task_name='pick best model',\n",
" parameter_override={'General/train_tasks_ids': '[${train_1.id}, ${train_2.id}]'}) "
"TABULAR_DATASET_ID = Task.get_task(task_name=\"Download and split tabular dataset\", project_name=\"Tabular Example\").id\n",
"\n",
"pipe = PipelineController( \n",
" project=\"Tabular Example\",\n",
" name=\"tabular training pipeline\", \n",
" add_pipeline_tags=True, \n",
" version=\"0.1\"\n",
")\n",
"pipe.set_default_execution_queue(default_execution_queue=\"default\")\n",
"pipe.add_step(\n",
" name=\"preprocessing_1\",\n",
" base_task_project=\"Tabular Example\",\n",
" base_task_name=\"tabular preprocessing\",\n",
" parameter_override={\n",
" \"General/data_task_id\": TABULAR_DATASET_ID,\n",
" \"General/fill_categorical_NA\": \"True\",\n",
" \"General/fill_numerical_NA\": \"True\",\n",
" },\n",
")\n",
"\n",
"pipe.add_step(\n",
" name=\"preprocessing_2\",\n",
" base_task_project=\"Tabular Example\",\n",
" base_task_name=\"tabular preprocessing\",\n",
" parameter_override={\n",
" \"General/data_task_id\": TABULAR_DATASET_ID,\n",
" \"General/fill_categorical_NA\": \"False\",\n",
" \"General/fill_numerical_NA\": \"True\",\n",
" },\n",
")\n",
"pipe.add_step(\n",
" name=\"train_1\",\n",
" parents=[\"preprocessing_1\"],\n",
" base_task_project=\"Tabular Example\",\n",
" base_task_name=\"tabular prediction\",\n",
" parameter_override={\"General/data_task_id\": \"${preprocessing_1.id}\"},\n",
")\n",
"pipe.add_step(\n",
" name=\"train_2\",\n",
" parents=[\"preprocessing_2\"],\n",
" base_task_project=\"Tabular Example\",\n",
" base_task_name=\"tabular prediction\",\n",
" parameter_override={\"General/data_task_id\": \"${preprocessing_2.id}\"},\n",
")\n",
"\n",
"pipe.add_step(\n",
" name=\"pick_best\",\n",
" parents=[\"train_1\", \"train_2\"],\n",
" base_task_project=\"Tabular Example\",\n",
" base_task_name=\"pick best model\",\n",
" parameter_override={\"General/train_tasks_ids\": \"[${train_1.id}, ${train_2.id}]\"},\n",
")"
]
},
{
@ -89,7 +112,7 @@
"name": "python",
"nbconvert_exporter": "python",
"pygments_lexer": "ipython3",
"version": "3.7.4"
"version": "3.8.5"
}
},
"nbformat": 4,