mirror of
https://github.com/clearml/clearml
synced 2025-03-16 00:17:15 +00:00
Edit pipeline example (#494)
This commit is contained in:
parent
3d4b9c7127
commit
8f84c42f5d
@ -27,35 +27,58 @@
|
|||||||
"metadata": {},
|
"metadata": {},
|
||||||
"outputs": [],
|
"outputs": [],
|
||||||
"source": [
|
"source": [
|
||||||
"task = Task.init(project_name='Tabular Example', task_name='tabular training pipeline', task_type=Task.TaskTypes.controller)"
|
"TABULAR_DATASET_ID = Task.get_task(task_name=\"Download and split tabular dataset\", project_name=\"Tabular Example\").id\n",
|
||||||
]
|
"\n",
|
||||||
},
|
"pipe = PipelineController( \n",
|
||||||
{
|
" project=\"Tabular Example\",\n",
|
||||||
"cell_type": "code",
|
" name=\"tabular training pipeline\", \n",
|
||||||
"execution_count": null,
|
" add_pipeline_tags=True, \n",
|
||||||
"metadata": {},
|
" version=\"0.1\"\n",
|
||||||
"outputs": [],
|
")\n",
|
||||||
"source": [
|
"pipe.set_default_execution_queue(default_execution_queue=\"default\")\n",
|
||||||
"pipe = PipelineController(default_execution_queue='dan_queue', add_pipeline_tags=True)\n",
|
"pipe.add_step(\n",
|
||||||
"pipe.add_step(name='preprocessing_1', base_task_project='Tabular Example', base_task_name='tabular preprocessing',\n",
|
" name=\"preprocessing_1\",\n",
|
||||||
" parameter_override={'General/data_task_id': '39fbf86fc4a341359ac6df4aa70ff91b',\n",
|
" base_task_project=\"Tabular Example\",\n",
|
||||||
" 'General/fill_categorical_NA': 'True',\n",
|
" base_task_name=\"tabular preprocessing\",\n",
|
||||||
" 'General/fill_numerical_NA': 'True'})\n",
|
" parameter_override={\n",
|
||||||
"pipe.add_step(name='preprocessing_2', base_task_project='Tabular Example', base_task_name='tabular preprocessing',\n",
|
" \"General/data_task_id\": TABULAR_DATASET_ID,\n",
|
||||||
" parameter_override={'General/data_task_id': '39fbf86fc4a341359ac6df4aa70ff91b',\n",
|
" \"General/fill_categorical_NA\": \"True\",\n",
|
||||||
" 'General/fill_categorical_NA': 'False',\n",
|
" \"General/fill_numerical_NA\": \"True\",\n",
|
||||||
" 'General/fill_numerical_NA': 'True'})\n",
|
" },\n",
|
||||||
" \n",
|
")\n",
|
||||||
"pipe.add_step(name='train_1', parents=['preprocessing_1'],\n",
|
"\n",
|
||||||
" base_task_project='Tabular Example', base_task_name='tabular prediction',\n",
|
"pipe.add_step(\n",
|
||||||
" parameter_override={'General/data_task_id': '${preprocessing_1.id}'})\n",
|
" name=\"preprocessing_2\",\n",
|
||||||
"pipe.add_step(name='train_2', parents=['preprocessing_2'],\n",
|
" base_task_project=\"Tabular Example\",\n",
|
||||||
" base_task_project='Tabular Example', base_task_name='tabular prediction',\n",
|
" base_task_name=\"tabular preprocessing\",\n",
|
||||||
" parameter_override={'General/data_task_id': '${preprocessing_2.id}'})\n",
|
" parameter_override={\n",
|
||||||
" \n",
|
" \"General/data_task_id\": TABULAR_DATASET_ID,\n",
|
||||||
"pipe.add_step(name='pick_best', parents=['train_1', 'train_2'],\n",
|
" \"General/fill_categorical_NA\": \"False\",\n",
|
||||||
" base_task_project='Tabular Example', base_task_name='pick best model',\n",
|
" \"General/fill_numerical_NA\": \"True\",\n",
|
||||||
" parameter_override={'General/train_tasks_ids': '[${train_1.id}, ${train_2.id}]'}) "
|
" },\n",
|
||||||
|
")\n",
|
||||||
|
"pipe.add_step(\n",
|
||||||
|
" name=\"train_1\",\n",
|
||||||
|
" parents=[\"preprocessing_1\"],\n",
|
||||||
|
" base_task_project=\"Tabular Example\",\n",
|
||||||
|
" base_task_name=\"tabular prediction\",\n",
|
||||||
|
" parameter_override={\"General/data_task_id\": \"${preprocessing_1.id}\"},\n",
|
||||||
|
")\n",
|
||||||
|
"pipe.add_step(\n",
|
||||||
|
" name=\"train_2\",\n",
|
||||||
|
" parents=[\"preprocessing_2\"],\n",
|
||||||
|
" base_task_project=\"Tabular Example\",\n",
|
||||||
|
" base_task_name=\"tabular prediction\",\n",
|
||||||
|
" parameter_override={\"General/data_task_id\": \"${preprocessing_2.id}\"},\n",
|
||||||
|
")\n",
|
||||||
|
"\n",
|
||||||
|
"pipe.add_step(\n",
|
||||||
|
" name=\"pick_best\",\n",
|
||||||
|
" parents=[\"train_1\", \"train_2\"],\n",
|
||||||
|
" base_task_project=\"Tabular Example\",\n",
|
||||||
|
" base_task_name=\"pick best model\",\n",
|
||||||
|
" parameter_override={\"General/train_tasks_ids\": \"[${train_1.id}, ${train_2.id}]\"},\n",
|
||||||
|
")"
|
||||||
]
|
]
|
||||||
},
|
},
|
||||||
{
|
{
|
||||||
@ -89,7 +112,7 @@
|
|||||||
"name": "python",
|
"name": "python",
|
||||||
"nbconvert_exporter": "python",
|
"nbconvert_exporter": "python",
|
||||||
"pygments_lexer": "ipython3",
|
"pygments_lexer": "ipython3",
|
||||||
"version": "3.7.4"
|
"version": "3.8.5"
|
||||||
}
|
}
|
||||||
},
|
},
|
||||||
"nbformat": 4,
|
"nbformat": 4,
|
||||||
|
Loading…
Reference in New Issue
Block a user