ClearML/clearml
1
0
Fork 0
mirror of https://github.com/clearml/clearml synced 2025-03-16 00:17:15 +00:00
Code Issues Actions 1 Packages Projects Releases Wiki Activity

Edit pipeline example (#494)

Browse Source
This commit is contained in:
pollfly 2021-11-21 12:30:05 +02:00 committed by GitHub
parent 3d4b9c7127
commit 8f84c42f5d
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
1 changed files with 53 additions and 30 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

83
examples/frameworks/pytorch/notebooks/table/tabular_ml_pipeline.ipynb
View File

@ -27,35 +27,58 @@
"metadata": {}, "metadata": {},
"outputs": [], "outputs": [],
"source": [ "source": [
"task = Task.init(project_name='Tabular Example', task_name='tabular training pipeline', task_type=Task.TaskTypes.controller)" "TABULAR_DATASET_ID = Task.get_task(task_name=\"Download and split tabular dataset\", project_name=\"Tabular Example\").id\n",
] "\n",
}, "pipe = PipelineController( \n",
{ " project=\"Tabular Example\",\n",
"cell_type": "code", " name=\"tabular training pipeline\", \n",
"execution_count": null, " add_pipeline_tags=True, \n",
"metadata": {}, " version=\"0.1\"\n",
"outputs": [], ")\n",
"source": [ "pipe.set_default_execution_queue(default_execution_queue=\"default\")\n",
"pipe = PipelineController(default_execution_queue='dan_queue', add_pipeline_tags=True)\n", "pipe.add_step(\n",
"pipe.add_step(name='preprocessing_1', base_task_project='Tabular Example', base_task_name='tabular preprocessing',\n", " name=\"preprocessing_1\",\n",
" parameter_override={'General/data_task_id': '39fbf86fc4a341359ac6df4aa70ff91b',\n", " base_task_project=\"Tabular Example\",\n",
" 'General/fill_categorical_NA': 'True',\n", " base_task_name=\"tabular preprocessing\",\n",
" 'General/fill_numerical_NA': 'True'})\n", " parameter_override={\n",
"pipe.add_step(name='preprocessing_2', base_task_project='Tabular Example', base_task_name='tabular preprocessing',\n", " \"General/data_task_id\": TABULAR_DATASET_ID,\n",
" parameter_override={'General/data_task_id': '39fbf86fc4a341359ac6df4aa70ff91b',\n", " \"General/fill_categorical_NA\": \"True\",\n",
" 'General/fill_categorical_NA': 'False',\n", " \"General/fill_numerical_NA\": \"True\",\n",
" 'General/fill_numerical_NA': 'True'})\n", " },\n",
" \n", ")\n",
"pipe.add_step(name='train_1', parents=['preprocessing_1'],\n", "\n",
" base_task_project='Tabular Example', base_task_name='tabular prediction',\n", "pipe.add_step(\n",
" parameter_override={'General/data_task_id': '${preprocessing_1.id}'})\n", " name=\"preprocessing_2\",\n",
"pipe.add_step(name='train_2', parents=['preprocessing_2'],\n", " base_task_project=\"Tabular Example\",\n",
" base_task_project='Tabular Example', base_task_name='tabular prediction',\n", " base_task_name=\"tabular preprocessing\",\n",
" parameter_override={'General/data_task_id': '${preprocessing_2.id}'})\n", " parameter_override={\n",
" \n", " \"General/data_task_id\": TABULAR_DATASET_ID,\n",
"pipe.add_step(name='pick_best', parents=['train_1', 'train_2'],\n", " \"General/fill_categorical_NA\": \"False\",\n",
" base_task_project='Tabular Example', base_task_name='pick best model',\n", " \"General/fill_numerical_NA\": \"True\",\n",
" parameter_override={'General/train_tasks_ids': '[${train_1.id}, ${train_2.id}]'}) " " },\n",
")\n",
"pipe.add_step(\n",
" name=\"train_1\",\n",
" parents=[\"preprocessing_1\"],\n",
" base_task_project=\"Tabular Example\",\n",
" base_task_name=\"tabular prediction\",\n",
" parameter_override={\"General/data_task_id\": \"${preprocessing_1.id}\"},\n",
")\n",
"pipe.add_step(\n",
" name=\"train_2\",\n",
" parents=[\"preprocessing_2\"],\n",
" base_task_project=\"Tabular Example\",\n",
" base_task_name=\"tabular prediction\",\n",
" parameter_override={\"General/data_task_id\": \"${preprocessing_2.id}\"},\n",
")\n",
"\n",
"pipe.add_step(\n",
" name=\"pick_best\",\n",
" parents=[\"train_1\", \"train_2\"],\n",
" base_task_project=\"Tabular Example\",\n",
" base_task_name=\"pick best model\",\n",
" parameter_override={\"General/train_tasks_ids\": \"[${train_1.id}, ${train_2.id}]\"},\n",
")"
] ]
}, },
{ {
@ -89,7 +112,7 @@
"name": "python", "name": "python",
"nbconvert_exporter": "python", "nbconvert_exporter": "python",
"pygments_lexer": "ipython3", "pygments_lexer": "ipython3",
"version": "3.7.4" "version": "3.8.5"
} }
}, },
"nbformat": 4, "nbformat": 4,