Graeme Malcolm (GraemeMalcolm) · Feb 14, 2020 · Feb 13, 2020 · Feb 14, 2020 · Feb 14, 2020 · Feb 14, 2020
diff --git a/diabetes_regression/register/register_model.py b/diabetes_regression/register/register_model.py
@@ -27,6 +27,7 @@
 import sys
 import argparse
 import traceback
+import joblib
 from azureml.core import Run, Experiment, Workspace
 from azureml.core.model import Model as AMLModel
 
@@ -74,6 +75,11 @@ def main():
         help="Name of the Model",
         default="sklearn_regression_model.pkl",
     )
+    parser.add_argument(
+        "--step_input",
+        type=str,
+        help=("input from previous steps")
+    )
 
     args = parser.parse_args()
     if (args.build_id is not None):
@@ -83,18 +89,39 @@ def main():
     if (run_id == 'amlcompute'):
         run_id = run.parent.id
     model_name = args.model_name
+    model_path = args.step_input
 
-    if (build_id is None):
-        register_aml_model(model_name, exp, run_id)
-    else:
-        run.tag("BuildId", value=build_id)
-        builduri_base = os.environ.get("BUILDURI_BASE")
-        if (builduri_base is not None):
-            build_uri = builduri_base + build_id
-            run.tag("BuildUri", value=build_uri)
-            register_aml_model(model_name, exp, run_id, build_id, build_uri)
+    # load the model
+    print("Loading model from " + model_path)
+    model_file = os.path.join(model_path, model_name)
+    model = joblib.load(model_file)
+
+    if (model is not None):
+        if (build_id is None):
+            register_aml_model(model_file, model_name, exp, run_id)
         else:
-            register_aml_model(model_name, exp, run_id, build_id)
+            run.tag("BuildId", value=build_id)
+            builduri_base = os.environ.get("BUILDURI_BASE")
+            if (builduri_base is not None):
+                build_uri = builduri_base + build_id
+                run.tag("BuildUri", value=build_uri)
+                register_aml_model(
+                    model_file,
+                    model_name,
+                    exp,
+                    run_id,
+                    build_id,
+                    build_uri)
+            else:
+                register_aml_model(
+                    model_file,
+                    model_name,
+                    exp,
+                    run_id,
+                    build_id)
+    else:
+        print("Model not found. Skipping model registration.")
+        sys.exit(0)
 
 
 def model_already_registered(model_name, exp, run_id):
@@ -109,35 +136,28 @@ def model_already_registered(model_name, exp, run_id):
 
 
 def register_aml_model(
+    model_path,
     model_name,
     exp,
     run_id,
     build_id: str = 'none',
     build_uri=None
 ):
     try:
-        if (build_id != 'none'):
-            model_already_registered(model_name, exp, run_id)
-            run = Run(experiment=exp, run_id=run_id)
-            tagsValue = {"area": "diabetes_regression",
-                         "BuildId": build_id, "run_id": run_id,
-                         "experiment_name": exp.name}
-            if (build_uri is not None):
-                tagsValue["BuildUri"] = build_uri
-        else:
-            run = Run(experiment=exp, run_id=run_id)
-            if (run is not None):
-                tagsValue = {"area": "diabetes_regression",
-                             "run_id": run_id, "experiment_name": exp.name}
-            else:
-                print("A model run for experiment", exp.name,
-                      "matching properties run_id =", run_id,
-                      "was not found. Skipping model registration.")
-                sys.exit(0)
+        model_already_registered(model_name, exp, run_id)
+        tagsValue = {
+            "area": "diabetes_regression",
+            "BuildId": build_id,
+            "run_id": run_id,
+            "experiment_name": exp.name}
+        if (build_uri is not None):
+            tagsValue["BuildUri"] = build_uri
 
-        model = run.register_model(model_name=model_name,
-                                   model_path="./outputs/" + model_name,
-                                   tags=tagsValue)
+        model = AMLModel.register(
+            workspace=exp.workspace,
+            model_name=model_name,
+            model_path=model_path,
+            tags=tagsValue)
         os.chdir("..")
         print(
             "Model registered: {} \nModel Description: {} "

diff --git a/diabetes_regression/training/train.py b/diabetes_regression/training/train.py
@@ -24,7 +24,6 @@
 POSSIBILITY OF SUCH DAMAGE.
 """
 from azureml.core.run import Run
-from azureml.core import Dataset
 import os
 import argparse
 from sklearn.datasets import load_diabetes
@@ -65,19 +64,20 @@ def main():
     )
 
     parser.add_argument(
-        "--dataset_name",
+        "--step_output",
         type=str,
-        help=("Dataset with the training data")
+        help=("output for passing data to next step")
     )
+
     args = parser.parse_args()
 
     print("Argument [build_id]: %s" % args.build_id)
     print("Argument [model_name]: %s" % args.model_name)
-    print("Argument [dataset_name]: %s" % args.dataset_name)
+    print("Argument [step_output]: %s" % args.step_output)
 
     model_name = args.model_name
     build_id = args.build_id
-    dataset_name = args.dataset_name
+    step_output_path = args.step_output
 
     print("Getting training parameters")
 
@@ -91,10 +91,10 @@ def main():
     print("Parameter alpha: %s" % alpha)
 
     run = Run.get_context()
-    ws = run.experiment.workspace
 
-    if (dataset_name):
-        dataset = Dataset.get_by_name(workspace=ws, name=dataset_name)
+    # Get the dataset
+    dataset = run.input_datasets['training_data']
+    if (dataset):
         df = dataset.to_pandas_dataframe()
         X = df.values
         y = df.Y
@@ -108,21 +108,18 @@ def main():
 
     reg = train_model(run, data, alpha)
 
-    joblib.dump(value=reg, filename=model_name)
-
-    # upload model file explicitly into artifacts for parent run
-    run.parent.upload_file(name="./outputs/" + model_name,
-                           path_or_stream=model_name)
-    print("Uploaded the model {} to experiment {}".format(
-        model_name, run.experiment.name))
-    dirpath = os.getcwd()
-    print(dirpath)
-    print("Following files are uploaded ")
-    print(run.parent.get_file_names())
+    # Pass model file to next step
+    os.makedirs(step_output_path, exist_ok=True)
+    output_path = os.path.join(step_output_path, model_name)
+    joblib.dump(value=reg, filename=output_path)
 
-    run.parent.tag("BuildId", value=build_id)
+    # Also upload model file to run outputs
+    os.makedirs('outputs', exist_ok=True)
+    output_path = os.path.join('outputs', model_name)
+    joblib.dump(value=reg, filename=output_path)
 
     # Add properties to identify this specific training run
+    run.parent.tag("BuildId", value=build_id)
     run.tag("BuildId", value=build_id)
     run.tag("run_type", value="train")
     builduri_base = os.environ.get("BUILDURI_BASE")

diff --git a/docs/getting_started.md b/docs/getting_started.md
@@ -60,6 +60,8 @@ The variable group should contain the following required variables:
 | WORKSPACE_NAME           | mlops-AML-WS             |
 | WORKSPACE_SVC_CONNECTION | aml-workspace-connection |
 | ACI_DEPLOYMENT_NAME      | diabetes-aci             |
+| DATASET_NAME      | diabetes_training             |
+| MODEL_NAME      | diabetes_model.pkl             |
 
 **Note:**
 
@@ -75,6 +77,12 @@ the BASE_NAME value should not exceed 10 characters and it should contain number
 
 The **RESOURCE_GROUP** parameter is used as the name for the resource group that will hold the Azure resources for the solution. If providing an existing AML Workspace, set this value to the corresponding resource group name.
 
+The **WORKSPACE_SVC_CONNECTION** parameter is used to reference a service connection for the Azure ML workspace. You will create this after provisioning the workspace (we recommend using the IaC pipeline as described below), and installing the Azure ML extension in your Azure DevOps project.
+
+The **DATASET_NAME** parameter is used to reference the training dataset that you will register in your Azure ML workspace.
+
+The **MODEL_NAME** parameter is used to specify the name (and filename) of the model that you will train.
+
 Make sure to select the **Allow access to all pipelines** checkbox in the
 variable group configuration.
 
@@ -125,8 +133,8 @@ Check out the newly created resources in the [Azure Portal](https://portal.azure
 
 (Optional) To remove the resources created for this project you can use the [/environment_setup/iac-remove-environment.yml](../environment_setup/iac-remove-environment.yml) definition or you can just delete the resource group in the [Azure Portal](https://portal.azure.com).
 
-**Note:** The training ML pipeline uses a [sample diabetes dataset](https://scikit-learn.org/stable/modules/generated/sklearn.datasets.load_diabetes.html) as training data. If you want to use your own dataset, you need to [create and register a datastore](https://docs.microsoft.com/en-us/azure/machine-learning/how-to-access-data#azure-machine-learning-studio) in your ML workspace and upload the datafile (e.g. [diabetes.csv](./data/diabetes.csv)) to the corresponding blob container. You can also define a datastore in the ML Workspace with [az cli](https://docs.microsoft.com/en-us/cli/azure/ext/azure-cli-ml/ml/datastore?view=azure-cli-latest#ext-azure-cli-ml-az-ml-datastore-attach-blob).
-You'll also need to configure DATASTORE_NAME and DATAFILE_NAME variables in ***devopsforai-aml-vg*** variable group.
+**Note:** The training ML pipeline uses a [sample diabetes dataset](https://scikit-learn.org/stable/modules/generated/sklearn.datasets.load_diabetes.html) as training data. To use this, you need to create a **Dataset** in your workspace by uoloading the diabetes.csv file, naming the dataset to match the DATASET_NAME variable you created earlier in ***devopsforai-aml-vg*** variable group. You can upload the data file to the default datastore for simplicity, but best practice is to [create and register a datastore](https://docs.microsoft.com/en-us/azure/machine-learning/how-to-access-data#azure-machine-learning-studio) in your ML workspace and upload the datafile to the corresponding blob container. You can also define a datastore in the ML Workspace with [az cli](https://docs.microsoft.com/en-us/cli/azure/ext/azure-cli-ml/ml/datastore?view=azure-cli-latest#ext-azure-cli-ml-az-ml-datastore-attach-blob).
+You can also configure DATASTORE_NAME and DATAFILE_NAME variables in ***devopsforai-aml-vg*** variable group to have the pipeline automatically upload the data file to your datastore and register a dataset.
 
 ## Create an Azure DevOps Azure ML Workspace Service Connection
 

diff --git a/ml_service/pipelines/diabetes_regression_build_train_pipeline.py b/ml_service/pipelines/diabetes_regression_build_train_pipeline.py
@@ -1,6 +1,6 @@
 from azureml.pipeline.core.graph import PipelineParameter
 from azureml.pipeline.steps import PythonScriptStep
-from azureml.pipeline.core import Pipeline
+from azureml.pipeline.core import Pipeline, PipelineData
 from azureml.core import Workspace, Environment
 from azureml.core.runconfig import RunConfiguration
 from azureml.core import Dataset, Datastore
@@ -46,25 +46,42 @@ def main():
         name="build_id", default_value=e.build_id)
 
     dataset_name = ""
-    if (e.datastore_name is not None and e.datafile_name is not None):
+    # Allow an existing dataset to be specified
+    if e.dataset_name is not None:
         dataset_name = e.dataset_name
-        datastore = Datastore.get(aml_workspace, e.datastore_name)
-        data_path = [(datastore, e.datafile_name)]
-        dataset = Dataset.Tabular.from_delimited_files(path=data_path)
-        dataset.register(workspace=aml_workspace,
-                         name=e.dataset_name,
-                         description="dataset with training data",
-                         create_new_version=True)
+        # If a datastore and file are also provided, create a new dataset
+        if (e.datastore_name is not None and e.datafile_name is not None):
+            # There's an assumption that a datset consists of a single file
+            # A dataset can consist of multiple files in a folder hierarchy
+            datastore = Datastore.get(aml_workspace, e.datastore_name)
+            data_path = [(datastore, e.datafile_name)]
+            # There's an assumption the dataset is tabular(could be file-based)
+            dataset = Dataset.Tabular.from_delimited_files(path=data_path)
+            dataset.register(
+                workspace=aml_workspace,
+                name=e.dataset_name,
+                description="dataset with training data",
+                create_new_version=True)
+
+    # Get the dataset
+    dataset = Dataset.get_by_name(aml_workspace, dataset_name)
+
+    # Create a PipelineData to pass data between steps
+    pipeline_data = PipelineData(
+        'pipeline_data',
+        datastore=aml_workspace.get_default_datastore())
 
     train_step = PythonScriptStep(
         name="Train Model",
         script_name=e.train_script_path,
         compute_target=aml_compute,
         source_directory=e.sources_directory_train,
+        inputs=[dataset.as_named_input('training_data')],
+        outputs=[pipeline_data],
         arguments=[
             "--build_id", build_id_param,
             "--model_name", model_name_param,
-            "--dataset_name", dataset_name,
+            "--step_output", pipeline_data
         ],
         runconfig=run_config,
         allow_reuse=False,
@@ -91,9 +108,11 @@ def main():
         script_name=e.register_script_path,
         compute_target=aml_compute,
         source_directory=e.sources_directory_train,
+        inputs=[pipeline_data],
         arguments=[
             "--build_id", build_id_param,
             "--model_name", model_name_param,
+            "--step_input", pipeline_data,
         ],
         runconfig=run_config,
         allow_reuse=False,