10_mlops/10a_mlpipeline.ipynb (535 lines of code) (raw):
{
"cells": [
{
"cell_type": "code",
"execution_count": 1,
"metadata": {
"colab": {
"base_uri": "https://localhost:8080/",
"height": 72
},
"id": "hiQ6zAoYhyaA",
"outputId": "0acee878-1207-42c3-9bee-a594acd44365"
},
"outputs": [
{
"data": {
"text/markdown": [
"\n",
"<table class=\"tfo-notebook-buttons\" align=\"left\">\n",
" <td>\n",
" <a target=\"_blank\" href=\"https://console.cloud.google.com/ai-platform/notebooks/deploy-notebook?name=ML Pipeline&url=https%3A%2F%2Fgithub.com%2FGoogleCloudPlatform%2Fpractical-ml-vision-book%2Fblob%2Fmaster%2F10_mlops%2F10a_mlpipeline.ipynb&download_url=https%3A%2F%2Fgithub.com%2FGoogleCloudPlatform%2Fpractical-ml-vision-book%2Fraw%2Fmaster%2F10_mlops%2F10a_mlpipeline.ipynb\">\n",
" <img src=\"https://raw.githubusercontent.com/GoogleCloudPlatform/practical-ml-vision-book/master/logo-cloud.png\"/> Run in AI Platform Notebook</a>\n",
" </td>\n",
" </td>\n",
" <td>\n",
" <a target=\"_blank\" href=\"https://colab.research.google.com/github/GoogleCloudPlatform/practical-ml-vision-book/blob/master/10_mlops/10a_mlpipeline.ipynb\">\n",
" <img src=\"https://www.tensorflow.org/images/colab_logo_32px.png\" />Run in Google Colab</a>\n",
" </td>\n",
" <td>\n",
" <a target=\"_blank\" href=\"https://github.com/GoogleCloudPlatform/practical-ml-vision-book/blob/master/10_mlops/10a_mlpipeline.ipynb\">\n",
" <img src=\"https://www.tensorflow.org/images/GitHub-Mark-32px.png\" />View source on GitHub</a>\n",
" </td>\n",
" <td>\n",
" <a href=\"https://raw.githubusercontent.com/GoogleCloudPlatform/practical-ml-vision-book/master/10_mlops/10a_mlpipeline.ipynb\">\n",
" <img src=\"https://www.tensorflow.org/images/download_logo_32px.png\" />Download notebook</a>\n",
" </td>\n",
"</table>\n"
],
"text/plain": [
"<IPython.core.display.Markdown object>"
]
},
"execution_count": 1,
"metadata": {},
"output_type": "execute_result"
}
],
"source": [
"from IPython.display import Markdown as md\n",
"\n",
"### change to reflect your notebook\n",
"_nb_loc = \"10_mlops/10a_mlpipeline.ipynb\"\n",
"_nb_title = \"ML Pipeline\"\n",
"\n",
"### no need to change any of this\n",
"_nb_safeloc = _nb_loc.replace('/', '%2F')\n",
"md(\"\"\"\n",
"<table class=\"tfo-notebook-buttons\" align=\"left\">\n",
" <td>\n",
" <a target=\"_blank\" href=\"https://console.cloud.google.com/ai-platform/notebooks/deploy-notebook?name={1}&url=https%3A%2F%2Fgithub.com%2FGoogleCloudPlatform%2Fpractical-ml-vision-book%2Fblob%2Fmaster%2F{2}&download_url=https%3A%2F%2Fgithub.com%2FGoogleCloudPlatform%2Fpractical-ml-vision-book%2Fraw%2Fmaster%2F{2}\">\n",
" <img src=\"https://raw.githubusercontent.com/GoogleCloudPlatform/practical-ml-vision-book/master/logo-cloud.png\"/> Run in AI Platform Notebook</a>\n",
" </td>\n",
" </td>\n",
" <td>\n",
" <a target=\"_blank\" href=\"https://colab.research.google.com/github/GoogleCloudPlatform/practical-ml-vision-book/blob/master/{0}\">\n",
" <img src=\"https://www.tensorflow.org/images/colab_logo_32px.png\" />Run in Google Colab</a>\n",
" </td>\n",
" <td>\n",
" <a target=\"_blank\" href=\"https://github.com/GoogleCloudPlatform/practical-ml-vision-book/blob/master/{0}\">\n",
" <img src=\"https://www.tensorflow.org/images/GitHub-Mark-32px.png\" />View source on GitHub</a>\n",
" </td>\n",
" <td>\n",
" <a href=\"https://raw.githubusercontent.com/GoogleCloudPlatform/practical-ml-vision-book/master/{0}\">\n",
" <img src=\"https://www.tensorflow.org/images/download_logo_32px.png\" />Download notebook</a>\n",
" </td>\n",
"</table>\n",
"\"\"\".format(_nb_loc, _nb_title, _nb_safeloc))"
]
},
{
"cell_type": "markdown",
"metadata": {
"id": "a8HQYsAtC0Fv"
},
"source": [
"# Machine Learning Pipeline\n",
"\n",
"In this notebook, we show how to run the flowers classification workflow as a pipeline"
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"## Set up"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": [
"%pip install --upgrade --user kfp"
]
},
{
"cell_type": "code",
"execution_count": 1,
"metadata": {},
"outputs": [],
"source": [
"# CHANGE AS needed\n",
"REGION = 'us-central1' # Change as needed to a region where you have quota\n",
"KFPHOST = 'https://40e09ee3a33a422-dot-us-central1.pipelines.googleusercontent.com' # Note name of launched Kubeflow Pipelines cluster"
]
},
{
"cell_type": "code",
"execution_count": 3,
"metadata": {},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"ai-analytics-solutions\n",
"env: PROJECT=ai-analytics-solutions\n",
"env: REGION=us-central1\n",
"env: BUCKET=ai-analytics-solutions-flowers-pipeline\n"
]
}
],
"source": [
"PROJECT = !gcloud config get-value project\n",
"PROJECT = PROJECT[0]\n",
"print(PROJECT)\n",
"%env PROJECT = {PROJECT}\n",
"%env REGION = {REGION}\n",
"BUCKET = PROJECT + \"-flowers-pipeline\"\n",
"%env BUCKET = {BUCKET}"
]
},
{
"cell_type": "code",
"execution_count": 3,
"metadata": {},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"Creating gs://ai-analytics-solutions-flowers-pipeline/...\n"
]
}
],
"source": [
"!gsutil mb -l {REGION} gs://{BUCKET}"
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"Build the container"
]
},
{
"cell_type": "code",
"execution_count": 4,
"metadata": {},
"outputs": [],
"source": [
"%%capture --no-stderr\n",
"!../build_docker_image.sh"
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"## Convert JPEG files to TF Records"
]
},
{
"cell_type": "code",
"execution_count": 5,
"metadata": {},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"Overwriting components/create_dataset.yaml\n"
]
}
],
"source": [
"%%writefile components/create_dataset.yaml\n",
"name: create_dataset\n",
"description: Converts JPEG files to TensorFlow Records using Dataflow or Apache Beam\n",
"inputs:\n",
"- {name: runner, type: str, default: 'DirectRunner', description: 'DirectRunner or DataflowRunner'}\n",
"- {name: project_id, type: str, description: 'Project to bill Dataflow job to'}\n",
"- {name: region, type: str, description: 'Region to run Dataflow job in'}\n",
"- {name: input_csv, type: GCSPath, description: 'Path to CSV file'}\n",
"- {name: output_dir, type: GCSPath, description: 'Top-level directory for TF records'}\n",
"- {name: labels_dict, type: GCSPath, description: 'Dictionary file for class names'}\n",
"outputs:\n",
"- {name: tfrecords_topdir, type: GCSPath, description: 'Top-level directory for TF records'}\n",
"implementation:\n",
" container:\n",
" image: gcr.io/ai-analytics-solutions/practical-ml-vision-book:latest\n",
" command: [\n",
" \"bash\", \"/src/practical-ml-vision-book/10_mlops/components/create_dataset.sh\"\n",
" ]\n",
" args: [\n",
" {inputValue: output_dir},\n",
" {outputPath: tfrecords_topdir},\n",
" \"--all_data\", {inputValue: input_csv},\n",
" \"--labels_file\", {inputValue: labels_dict},\n",
" \"--project_id\", {inputValue: project_id},\n",
" \"--output_dir\", {inputValue: output_dir},\n",
" \"--runner\", {inputValue: runner},\n",
" \"--region\", {inputValue: region},\n",
" ]"
]
},
{
"cell_type": "code",
"execution_count": 6,
"metadata": {},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"Overwriting components/noop_create_dataset.yaml\n"
]
}
],
"source": [
"%%writefile components/noop_create_dataset.yaml\n",
"name: noop_create_dataset\n",
"description: Converts JPEG files to TensorFlow Records using Dataflow or Apache Beam\n",
"inputs:\n",
"- {name: runner, type: str, default: 'DirectRunner', description: 'DirectRunner or DataflowRunner'}\n",
"- {name: project_id, type: str, description: 'Project to bill Dataflow job to'}\n",
"- {name: region, type: str, description: 'Region to run Dataflow job in'}\n",
"- {name: input_csv, type: GCSPath, description: 'Path to CSV file'}\n",
"- {name: output_dir, type: GCSPath, description: 'Top-level directory for TF records'}\n",
"- {name: labels_dict, type: GCSPath, description: 'Dictionary file for class names'}\n",
"outputs:\n",
"- {name: tfrecords_topdir, type: GCSPath, description: 'Top-level directory for TF records'}\n",
"implementation:\n",
" container:\n",
" image: gcr.io/ai-analytics-solutions/practical-ml-vision-book:latest\n",
" command: [\n",
" \"bash\", \"/src/practical-ml-vision-book/10_mlops/components/noop_create_dataset.sh\"\n",
" ]\n",
" args: [\n",
" {inputValue: output_dir},\n",
" {outputPath: tfrecords_topdir}\n",
" ]"
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"## Train model\n",
"\n",
"To do it locally on the cluster instead of on CAIP, we'll use gcloud local training\n",
"\n",
"<pre>\n",
"gcloud ai-platform local train --package-path $PACKAGE_PATH \n",
" --module-name $MODULE_NAME --job-dir ${JOB_DIR}_local \n",
" -- --num_training_examples 100 --with_color_distort False --crop_ratio 0.6\n",
"</pre>"
]
},
{
"cell_type": "code",
"execution_count": 7,
"metadata": {},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"Overwriting components/train_model_kfp.yaml\n"
]
}
],
"source": [
"%%writefile components/train_model_kfp.yaml\n",
"name: train_model_kfp\n",
"description: Trains an ML model on KFP\n",
"inputs:\n",
"- {name: input_topdir, type: GCSPath, description: 'Top-level directory for TF records'}\n",
"- {name: region, type: str, description: 'Region (ignored)'}\n",
"- {name: job_dir, type: GCSPath, description: 'Top-level output directory'}\n",
"outputs:\n",
"- {name: trained_model, type: GCSPath, description: 'location of trained model'}\n",
"implementation:\n",
" container:\n",
" image: gcr.io/ai-analytics-solutions/practical-ml-vision-book:latest\n",
" command: [\n",
" \"bash\", \"/src/practical-ml-vision-book/10_mlops/components/train_model_kfp.sh\", \n",
" ]\n",
" args: [\n",
" {inputValue: input_topdir},\n",
" {inputValue: region},\n",
" {inputValue: job_dir},\n",
" {outputPath: trained_model},\n",
" ]"
]
},
{
"cell_type": "code",
"execution_count": 8,
"metadata": {},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"Overwriting components/train_model_caip.yaml\n"
]
}
],
"source": [
"%%writefile components/train_model_caip.yaml\n",
"name: train_model_caip\n",
"description: Trains an ML model on CAIP\n",
"inputs:\n",
"- {name: input_topdir, type: GCSPath, description: 'Top-level directory for TF records'}\n",
"- {name: region, type: str, description: 'Region'}\n",
"- {name: job_dir, type: GCSPath, description: 'Top-level output directory'}\n",
"outputs:\n",
"- {name: trained_model, type: GCSPath, description: 'location of trained model'}\n",
"implementation:\n",
" container:\n",
" image: gcr.io/ai-analytics-solutions/practical-ml-vision-book:latest\n",
" command: [\n",
" \"bash\", \"/src/practical-ml-vision-book/10_mlops/components/train_model_caip.sh\", \n",
" ]\n",
" args: [\n",
" {inputValue: input_topdir},\n",
" {inputValue: region},\n",
" {inputValue: job_dir},\n",
" {outputPath: trained_model},\n",
" ]"
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"## The pipeline"
]
},
{
"cell_type": "code",
"execution_count": 4,
"metadata": {},
"outputs": [],
"source": [
"import kfp\n",
"import kfp.dsl as dsl\n",
"import json\n",
"import os\n",
"\n",
"create_dataset_op = kfp.components.load_component_from_file(\n",
" #'components/noop_create_dataset.yaml'\n",
" 'components/create_dataset.yaml'\n",
")\n",
"train_model_op = kfp.components.load_component_from_file(\n",
" #'components/train_model_kfp.yaml'\n",
" 'components/train_model_caip.yaml'\n",
")\n",
"deploy_op = kfp.components.load_component_from_url(\n",
" 'https://raw.githubusercontent.com/kubeflow/pipelines/master/components/gcp/ml_engine/deploy/component.yaml')\n",
"\n",
"@dsl.pipeline(\n",
" name='Flowers Transfer Learning Pipeline',\n",
" description='End-to-end pipeline'\n",
")\n",
"def flowerstxf_pipeline(\n",
" project_id = PROJECT,\n",
" bucket = BUCKET,\n",
" region = REGION\n",
"):\n",
" # Step 1: Create dataset\n",
" create_dataset = create_dataset_op(\n",
" runner='DataflowRunner',\n",
" project_id=project_id,\n",
" region=region,\n",
" input_csv='gs://practical-ml-vision-book-data/flowers_5_jpeg/flower_photos/all_data.csv',\n",
" output_dir='gs://{}/data/flower_tfrecords'.format(bucket),\n",
" labels_dict='gs://practical-ml-vision-book-data/flowers_5_jpeg/flower_photos/dict.txt'\n",
" )\n",
" create_dataset.execution_options.caching_strategy.max_cache_staleness = \"P7D\"\n",
" \n",
" # Step 2: Train model\n",
" train_model = train_model_op(\n",
" input_topdir=create_dataset.outputs['tfrecords_topdir'],\n",
" region=region,\n",
" job_dir='gs://{}/trained_model'.format(bucket)\n",
" )\n",
" train_model.execution_options.caching_strategy.max_cache_staleness = \"P0D\"\n",
" \n",
" # Step 3: Deploy trained model\n",
" deploy_model = deploy_op(\n",
" model_uri=train_model.outputs['trained_model'],\n",
" project_id=project_id, \n",
" model_id='flowers', \n",
" version_id='txf', \n",
" runtime_version='2.3', \n",
" python_version='3.7',\n",
" version={}, \n",
" replace_existing_version='True', \n",
" set_default='True', \n",
" wait_interval='30')"
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"## Compile and submit pipeline"
]
},
{
"cell_type": "code",
"execution_count": 5,
"metadata": {},
"outputs": [],
"source": [
"pipeline_func = flowerstxf_pipeline\n",
"pipeline_filename = pipeline_func.__name__ + '.zip'\n",
"import kfp.compiler as compiler\n",
"compiler.Compiler().compile(pipeline_func, pipeline_filename)"
]
},
{
"cell_type": "code",
"execution_count": 6,
"metadata": {},
"outputs": [
{
"data": {
"text/html": [
"<a href=\"https://40e09ee3a33a422-dot-us-central1.pipelines.googleusercontent.com/#/experiments/details/215610c4-312c-426a-aef3-1f8b1ccb7cb4\" target=\"_blank\" >Experiment details</a>."
],
"text/plain": [
"<IPython.core.display.HTML object>"
]
},
"metadata": {},
"output_type": "display_data"
},
{
"data": {
"text/html": [
"<a href=\"https://40e09ee3a33a422-dot-us-central1.pipelines.googleusercontent.com/#/runs/details/2d66a092-3648-4171-b77f-22cd0dcaba1c\" target=\"_blank\" >Run details</a>."
],
"text/plain": [
"<IPython.core.display.HTML object>"
]
},
"metadata": {},
"output_type": "display_data"
}
],
"source": [
"import kfp\n",
"client = kfp.Client(host=KFPHOST)\n",
"experiment = client.create_experiment('from_notebook')\n",
"run_name = pipeline_func.__name__ + ' run'\n",
"run_result = client.run_pipeline(experiment.id, run_name, pipeline_filename, \n",
" {\n",
" 'project_id': PROJECT,\n",
" 'bucket': BUCKET,\n",
" 'region': REGION\n",
" })"
]
},
{
"cell_type": "markdown",
"metadata": {
"id": "Duu8mX3iXANE"
},
"source": [
"## License\n",
"Copyright 2020 Google Inc. Licensed under the Apache License, Version 2.0 (the \"License\"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an \"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License."
]
}
],
"metadata": {
"accelerator": "GPU",
"colab": {
"collapsed_sections": [
"5UOm2etrwYCs"
],
"name": "03a_transfer_learning.ipynb",
"provenance": [],
"toc_visible": true
},
"environment": {
"name": "tf2-gpu.2-5.m75",
"type": "gcloud",
"uri": "gcr.io/deeplearning-platform-release/tf2-gpu.2-5:m75"
},
"kernelspec": {
"display_name": "Python 3",
"language": "python",
"name": "python3"
},
"language_info": {
"codemirror_mode": {
"name": "ipython",
"version": 3
},
"file_extension": ".py",
"mimetype": "text/x-python",
"name": "python",
"nbconvert_exporter": "python",
"pygments_lexer": "ipython3",
"version": "3.7.10"
}
},
"nbformat": 4,
"nbformat_minor": 4
}