Skip to content
This repository was archived by the owner on Sep 25, 2026. It is now read-only.

Commit 9bc9f8b

Browse files
author
j-so
committed
Squashed commit of the following:
commit 5af6eeb Author: j-so <jenns@microsoft.com> Date: Mon Jun 15 18:31:15 2020 -0700 fix bootstrap commit f61e103 Merge: 2796b40 08bb6f4 Author: j-so <jenns@microsoft.com> Date: Mon Jun 15 18:30:21 2020 -0700 Merge branch 'master' into jenns/splitpipeline_docfix commit 2796b40 Author: j-so <jenns@microsoft.com> Date: Mon Jun 15 18:30:00 2020 -0700 remove old pipeline commit 08bb6f4 Author: David Tesar <david.tesar@microsoft.com> Date: Mon Jun 15 14:09:12 2020 -0700 Simplify docs flow (#297) commit cd762ec Author: jotaylo <jotaylo@microsoft.com> Date: Mon Jun 15 12:28:23 2020 -0700 Move instruction to install AML extension to Azure Devops setup instructions (#298)
1 parent dbe8b33 commit 9bc9f8b

10 files changed

Lines changed: 105 additions & 397 deletions

.pipelines/diabetes_regression-cd-deploy.yml renamed to .pipelines/diabetes_regression-cd.yml

File renamed without changes.

‎.pipelines/diabetes_regression-ci-image.yml‎

Lines changed: 5 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -30,14 +30,9 @@ variables:
3030
value: 'scoring/scoreB.py'
3131

3232
steps:
33-
- task: AzureCLI@1
34-
inputs:
35-
azureSubscription: '$(WORKSPACE_SVC_CONNECTION)'
36-
scriptLocation: inlineScript
37-
workingDirectory: $(Build.SourcesDirectory)
38-
inlineScript: |
39-
set -e
40-
export SUBSCRIPTION_ID=$(az account show --query id -o tsv)
41-
python3 -m ml_service.util.create_scoring_image
42-
displayName: 'Create Scoring Image'
33+
- template: diabetes_regression-package-model-template.yml
34+
parameters:
35+
modelId: $(MODEL_NAME):$(MODEL_VERSION)
36+
scoringScriptPath: '$(Build.SourcesDirectory)/$(SOURCES_DIR_TRAIN)/$(SCORE_SCRIPT)'
37+
condaFilePath: '$(Build.SourcesDirectory)/$(SOURCES_DIR_TRAIN)/conda_dependencies.yml'
4338

‎.pipelines/diabetes_regression-ci-train-register.yml‎

Lines changed: 0 additions & 97 deletions
This file was deleted.

‎.pipelines/diabetes_regression-ci.yml‎

Lines changed: 5 additions & 114 deletions
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,4 @@
1-
# Continuous Integration (CI) pipeline that orchestrates the training, evaluation, registration, deployment, and testing of the diabetes_regression model.
1+
# Continuous Integration (CI) pipeline that orchestrates the training, evaluation, and registration of the diabetes_regression model.
22

33
resources:
44
containers:
@@ -27,7 +27,6 @@ pool:
2727
stages:
2828
- stage: 'Model_CI'
2929
displayName: 'Model CI'
30-
condition: not(variables['MODEL_BUILD_ID'])
3130
jobs:
3231
- job: "Model_CI_Pipeline"
3332
displayName: "Model CI Pipeline"
@@ -48,8 +47,8 @@ stages:
4847
displayName: 'Publish Azure Machine Learning Pipeline'
4948

5049
- stage: 'Trigger_AML_Pipeline'
51-
displayName: 'Train model'
52-
condition: and(succeeded(), not(variables['MODEL_BUILD_ID']))
50+
displayName: 'Train and evaluate model'
51+
condition: succeeded()
5352
variables:
5453
BUILD_URI: '$(SYSTEM.COLLECTIONURI)$(SYSTEM.TEAMPROJECT)/_build/results?buildId=$(BUILD.BUILDID)'
5554
jobs:
@@ -91,116 +90,8 @@ stages:
9190
- job: "Training_Run_Report"
9291
dependsOn: "Run_ML_Pipeline"
9392
condition: always()
94-
displayName: "Determine if evaluation succeeded and new model is registered"
93+
displayName: "Publish artifact if new model was registered"
9594
container: mlops
9695
timeoutInMinutes: 0
9796
steps:
98-
- template: diabetes_regression-get-model-version-template.yml
99-
100-
- stage: 'Deploy_ACI'
101-
displayName: 'Deploy to ACI'
102-
dependsOn: Trigger_AML_Pipeline
103-
condition: and(or(succeeded(), variables['MODEL_BUILD_ID']), variables['ACI_DEPLOYMENT_NAME'])
104-
jobs:
105-
- job: "Deploy_ACI"
106-
displayName: "Deploy to ACI"
107-
container: mlops
108-
timeoutInMinutes: 0
109-
steps:
110-
- template: diabetes_regression-get-model-version-template.yml
111-
- task: ms-air-aiagility.vss-services-azureml.azureml-model-deploy-task.AMLModelDeploy@0
112-
displayName: 'Azure ML Model Deploy'
113-
inputs:
114-
azureSubscription: $(WORKSPACE_SVC_CONNECTION)
115-
modelSourceType: manualSpec
116-
modelName: '$(MODEL_NAME)'
117-
modelVersion: $(MODEL_VERSION)
118-
inferencePath: '$(Build.SourcesDirectory)/$(SOURCES_DIR_TRAIN)/scoring/inference_config.yml'
119-
deploymentTarget: ACI
120-
deploymentName: $(ACI_DEPLOYMENT_NAME)
121-
deployConfig: '$(Build.SourcesDirectory)/$(SOURCES_DIR_TRAIN)/scoring/deployment_config_aci.yml'
122-
overwriteExistingDeployment: true
123-
- task: AzureCLI@1
124-
displayName: 'Smoke test'
125-
inputs:
126-
azureSubscription: '$(WORKSPACE_SVC_CONNECTION)'
127-
scriptLocation: inlineScript
128-
inlineScript: |
129-
set -e # fail on error
130-
export SUBSCRIPTION_ID=$(az account show --query id -o tsv)
131-
python -m ml_service.util.smoke_test_scoring_service --type ACI --service "$(ACI_DEPLOYMENT_NAME)"
132-
133-
- stage: 'Deploy_AKS'
134-
displayName: 'Deploy to AKS'
135-
dependsOn: Deploy_ACI
136-
condition: and(succeeded(), variables['AKS_DEPLOYMENT_NAME'])
137-
jobs:
138-
- job: "Deploy_AKS"
139-
displayName: "Deploy to AKS"
140-
container: mlops
141-
timeoutInMinutes: 0
142-
steps:
143-
- template: diabetes_regression-get-model-version-template.yml
144-
- task: ms-air-aiagility.vss-services-azureml.azureml-model-deploy-task.AMLModelDeploy@0
145-
displayName: 'Azure ML Model Deploy'
146-
inputs:
147-
azureSubscription: $(WORKSPACE_SVC_CONNECTION)
148-
modelSourceType: manualSpec
149-
modelName: '$(MODEL_NAME)'
150-
modelVersion: $(MODEL_VERSION)
151-
inferencePath: '$(Build.SourcesDirectory)/$(SOURCES_DIR_TRAIN)/scoring/inference_config.yml'
152-
deploymentTarget: AKS
153-
aksCluster: $(AKS_COMPUTE_NAME)
154-
deploymentName: $(AKS_DEPLOYMENT_NAME)
155-
deployConfig: '$(Build.SourcesDirectory)/$(SOURCES_DIR_TRAIN)/scoring/deployment_config_aks.yml'
156-
overwriteExistingDeployment: true
157-
- task: AzureCLI@1
158-
displayName: 'Smoke test'
159-
inputs:
160-
azureSubscription: '$(WORKSPACE_SVC_CONNECTION)'
161-
scriptLocation: inlineScript
162-
inlineScript: |
163-
set -e # fail on error
164-
export SUBSCRIPTION_ID=$(az account show --query id -o tsv)
165-
python -m ml_service.util.smoke_test_scoring_service --type AKS --service "$(AKS_DEPLOYMENT_NAME)"
166-
167-
- stage: 'Deploy_Webapp'
168-
displayName: 'Deploy to Webapp'
169-
dependsOn: Trigger_AML_Pipeline
170-
condition: and(or(succeeded(), variables['MODEL_BUILD_ID']), variables['WEBAPP_DEPLOYMENT_NAME'])
171-
jobs:
172-
- job: "Deploy_Webapp"
173-
displayName: "Deploy to Webapp"
174-
container: mlops
175-
timeoutInMinutes: 0
176-
steps:
177-
- template: diabetes_regression-get-model-version-template.yml
178-
- task: AzureCLI@1
179-
displayName: 'Create scoring image and set IMAGE_LOCATION variable'
180-
inputs:
181-
azureSubscription: '$(WORKSPACE_SVC_CONNECTION)'
182-
scriptLocation: inlineScript
183-
inlineScript: |
184-
set -e # fail on error
185-
export SUBSCRIPTION_ID=$(az account show --query id -o tsv)
186-
python -m ml_service.util.create_scoring_image --output_image_location_file image_location.txt
187-
# Output image location to Azure DevOps job
188-
IMAGE_LOCATION="$(cat image_location.txt)"
189-
echo "##vso[task.setvariable variable=IMAGE_LOCATION]$IMAGE_LOCATION"
190-
- task: AzureWebAppContainer@1
191-
name: WebAppDeploy
192-
displayName: 'Azure Web App on Container Deploy'
193-
inputs:
194-
azureSubscription: '$(AZURE_RM_SVC_CONNECTION)'
195-
appName: '$(WEBAPP_DEPLOYMENT_NAME)'
196-
resourceGroupName: '$(RESOURCE_GROUP)'
197-
imageName: '$(IMAGE_LOCATION)'
198-
- task: AzureCLI@1
199-
displayName: 'Smoke test'
200-
inputs:
201-
azureSubscription: '$(WORKSPACE_SVC_CONNECTION)'
202-
scriptLocation: inlineScript
203-
inlineScript: |
204-
set -e # fail on error
205-
export SUBSCRIPTION_ID=$(az account show --query id -o tsv)
206-
python -m ml_service.util.smoke_test_scoring_service --type Webapp --service "$(WebAppDeploy.AppServiceApplicationUrl)/score"
97+
- template: diabetes_regression-publish-model-artifact-template.yml

‎.pipelines/diabetes_regression-get-model-version-template.yml‎

Lines changed: 0 additions & 15 deletions
This file was deleted.

‎bootstrap/README.md‎

Lines changed: 1 addition & 16 deletions
Original file line numberDiff line numberDiff line change
@@ -1,18 +1,3 @@
11
# Bootstrap from MLOpsPython repository
22

3-
To use this existing project structure and scripts for your new ML project, you can quickly get started from the existing repository, bootstrap and create a template that works for your ML project.
4-
5-
Bootstrapping will prepare a directory structure for your project which includes:
6-
7-
* renaming files and folders from the base project name `diabetes_regression` to your project name
8-
* fixing imports and absolute path based on your project name
9-
* deleting and cleaning up some directories
10-
11-
To bootstrap from the existing MLOpsPython repository:
12-
13-
1. Ensure Python 3 is installed locally
14-
1. Clone this repository locally
15-
1. Run bootstrap.py script
16-
`python bootstrap.py -d [dirpath] -n [projectname]`
17-
* `[dirpath]` is the absolute path to the root of the directory where MLOpsPython is cloned
18-
* `[projectname]` is the name of your ML project
3+
For steps on how to use the bootstrap script, please see the "Bootstrap the project" section of the [custom model guide](../docs/custom_model.md#bootstrap-the-project).

‎bootstrap/bootstrap.py‎

Lines changed: 1 addition & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -84,14 +84,12 @@ def replace_project_name(project_dir, project_name, rename_name):
8484
files = [r".env.example",
8585
r".pipelines/code-quality-template.yml",
8686
r".pipelines/pr.yml",
87-
r".pipelines/diabetes_regression-cd-deploy.yml",
88-
r".pipelines/diabetes_regression-ci-train-register.yml",
87+
r".pipelines/diabetes_regression-cd.yml",
8988
r".pipelines/diabetes_regression-ci.yml",
9089
r".pipelines/abtest.yml",
9190
r".pipelines/diabetes_regression-ci-image.yml",
9291
r".pipelines/diabetes_regression-publish-model-artifact-template.yml", # NOQA: E501
9392
r".pipelines/diabetes_regression-get-model-id-artifact-template.yml", # NOQA: E501
94-
r".pipelines/diabetes_regression-get-model-version-template.yml", # NOQA: E501
9593
r".pipelines/diabetes_regression-variables-template.yml",
9694
r"environment_setup/Dockerfile",
9795
r"environment_setup/install_requirements.sh",

‎docs/custom_model.md‎

Lines changed: 18 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -3,11 +3,11 @@
33
This document provides steps to follow when using this repository as a template to train models and deploy the models with real-time inference in Azure ML with your own scripts and data.
44

55
1. Follow the MLOpsPython [Getting Started](getting_started.md) guide
6-
1. Follow the MLOpsPython [bootstrap instructions](../bootstrap/README.md) to create your project starting point
6+
1. Bootstrap the project
77
1. Configure training data
88
1. [If necessary] Convert your ML experimental code into production ready code
99
1. Replace the training code
10-
1. Update the evaluation code
10+
1. [Optional] Update the evaluation code
1111
1. Customize the build agent environment
1212
1. [If appropriate] Replace the score code
1313

@@ -17,24 +17,36 @@ Follow the [Getting Started](getting_started.md) guide to set up the infrastruct
1717

1818
Take a look at the [Repo Details](code_description.md) document for a description of the structure of this repository.
1919

20-
## Follow the Bootstrap instructions
20+
## Bootstrap the project
2121

22-
The [Bootstrap from MLOpsPython repository](../bootstrap/README.md) guide will help you to quickly prepare the repository for your project.
22+
Bootstrapping will prepare the directory structure to be used for your project name which includes:
23+
24+
* renaming files and folders from the base project name `diabetes_regression` to your project name
25+
* fixing imports and absolute path based on your project name
26+
* deleting and cleaning up some directories
2327

2428
**Note:** Since the bootstrap script will rename the `diabetes_regression` folder to the project name of your choice, we'll refer to your project as `[project name]` when paths are involved.
2529

30+
To bootstrap from the existing MLOpsPython repository:
31+
32+
1. Ensure Python 3 is installed locally
33+
1. From a local copy of the code, run the `bootstrap.py` script in the `bootstrap` folder
34+
`python bootstrap.py -d [dirpath] -n [projectname]`
35+
* `[dirpath]` is the absolute path to the root of the directory where MLOpsPython is cloned
36+
* `[projectname]` is the name of your ML project
37+
2638
## Configure training data
2739

2840
The training ML pipeline uses a [sample diabetes dataset](https://scikit-learn.org/stable/modules/generated/sklearn.datasets.load_diabetes.html) as training data.
2941

30-
To use your own data:
42+
**Important** Convert the template to use your own Azure ML Dataset for model training via these steps:
3143

3244
1. [Create a Dataset](https://docs.microsoft.com/azure/machine-learning/how-to-create-register-datasets) in your Azure ML workspace
3345
1. Update the `DATASET_NAME` and `DATASTORE_NAME` variables in `.pipelines/[project name]-variables-template.yml`
3446

3547
## Convert your ML experimental code into production ready code
3648

37-
The MLOpsPython template creates an Azure Machine Learning (ML) pipeline that invokes a set of [Azure ML pipeline steps](https://docs.microsoft.com/python/api/azureml-pipeline-steps/azureml.pipeline.steps) (see `ml_service/pipelines/[project name]_build_train_pipeline.py`). If your experiment is currently in a Jupyter notebook, it will need to be refactored into scripts that can be run independantly and dropped into the template which the existing Azure ML pipeline steps utilize.
49+
The MLOpsPython template creates an Azure Machine Learning (ML) pipeline that invokes a set of [Azure ML pipeline steps](https://docs.microsoft.com/python/api/azureml-pipeline-steps/azureml.pipeline.steps) (see `ml_service/pipelines/[project name]_build_train_pipeline.py`). If your experiment is currently in a Jupyter notebook, it will need to be refactored into scripts that can be run independently and dropped into the template which the existing Azure ML pipeline steps utilize.
3850

3951
1. Refactor your experiment code into scripts
4052
1. [Recommended] Prepare unit tests

0 commit comments

Comments
 (0)