Fixes the AML image snapshot path

mokarian · Jul 2, 2020 · Jul 2, 2020 · Jul 2, 2020 · Jul 2, 2020 · Jul 3, 2020
commit 56a20bff69320023cb6e7ce516df58b29c77e42d
diff --git a/.pipelines/diabetes_regression-cd.yml b/.pipelines/diabetes_regression-cd.yml
@@ -55,7 +55,7 @@ stages:
       inputs:
         azureSubscription: '$(WORKSPACE_SVC_CONNECTION)'
         scriptLocation: inlineScript
-        workingDirectory: $(Build.SourcesDirectory)/$(SOURCES_DIR_TRAIN)/scoring
+        workingDirectory: $(Build.SourcesDirectory)/$(SOURCES_DIR_TRAIN)/diabetes_regression/scoring
         inlineScript: |
           set -e # fail on error
 
@@ -101,7 +101,7 @@ stages:
       inputs:
         azureSubscription: '$(WORKSPACE_SVC_CONNECTION)'
         scriptLocation: inlineScript
-        workingDirectory: $(Build.SourcesDirectory)/$(SOURCES_DIR_TRAIN)/scoring
+        workingDirectory: $(Build.SourcesDirectory)/$(SOURCES_DIR_TRAIN)/diabetes_regression/scoring
         inlineScript: |
           set -e # fail on error
 
@@ -138,8 +138,8 @@ stages:
     - template: diabetes_regression-package-model-template.yml
       parameters:
         modelId: $(MODEL_NAME):$(MODEL_VERSION)
-        scoringScriptPath: '$(Build.SourcesDirectory)/$(SOURCES_DIR_TRAIN)/scoring/score.py'
-        condaFilePath: '$(Build.SourcesDirectory)/$(SOURCES_DIR_TRAIN)/conda_dependencies.yml'
+        scoringScriptPath: '$(Build.SourcesDirectory)/$(SOURCES_DIR_TRAIN)/diabetes_regression/scoring/score.py'
+        condaFilePath: '$(Build.SourcesDirectory)/$(SOURCES_DIR_TRAIN)/diabetes_regression/conda_dependencies.yml'
     - script: echo $(IMAGE_LOCATION) >image_location.txt
       displayName: "Write image location file"
     - task: AzureWebAppContainer@1

diff --git a/.pipelines/diabetes_regression-variables-template.yml b/.pipelines/diabetes_regression-variables-template.yml
@@ -3,19 +3,19 @@ variables:
   # Source Config
   # The directory containing the scripts for training, evaluating, and registering the model
   - name: SOURCES_DIR_TRAIN
-    value: diabetes_regression
+    value: .
     # The path to the model training script under SOURCES_DIR_TRAIN
   - name: TRAIN_SCRIPT_PATH
-    value: training/train_aml.py
+    value: diabetes_regression/training/train_aml.py
     # The path to the model evaluation script under SOURCES_DIR_TRAIN
   - name: EVALUATE_SCRIPT_PATH
-    value: evaluate/evaluate_model.py
+    value: diabetes_regression/evaluate/evaluate_model.py
     # The path to the model registration script under SOURCES_DIR_TRAIN
   - name: REGISTER_SCRIPT_PATH
-    value: register/register_model.py
+    value: diabetes_regression/register/register_model.py
     # The path to the model scoring script relative to SOURCES_DIR_TRAIN
   - name: SCORE_SCRIPT
-    value: scoring/score.py
+    value: diabetes_regression/scoring/score.py
 
 
   # Azure ML Variables

diff --git a/diabetes_regression/evaluate/evaluate_model.py b/diabetes_regression/evaluate/evaluate_model.py
@@ -26,7 +26,7 @@
 from azureml.core import Run
 import argparse
 import traceback
-from util.model_helper import get_model
+from diabetes_regression.util.model_helper import get_model
 
 run = Run.get_context()
 
@@ -42,7 +42,7 @@
 #     load_dotenv()
 #     sources_dir = os.environ.get("SOURCES_DIR_TRAIN")
 #     if (sources_dir is None):
-#         sources_dir = 'diabetes_regression'
+#         sources_dir = '.'
 #     path_to_util = os.path.join(".", sources_dir, "util")
 #     sys.path.append(os.path.abspath(path_to_util))  # NOQA: E402
 #     from model_helper import get_model

diff --git a/diabetes_regression/register/register_model.py b/diabetes_regression/register/register_model.py
@@ -90,7 +90,7 @@ def main():
     print("Getting registration parameters")
 
     # Load the registration parameters from the parameters file
-    with open("parameters.json") as f:
+    with open("diabetes_regression/parameters.json") as f:
         pars = json.load(f)
     try:
         register_args = pars["registration"]

diff --git a/diabetes_regression/scoring/parallel_batchscore.py b/diabetes_regression/scoring/parallel_batchscore.py
@@ -29,7 +29,7 @@
 import joblib
 import sys
 from typing import List
-from util.model_helper import get_model
+from diabetes_regression.util.model_helper import get_model
 from azureml.core import Model
 
 model = None

diff --git a/diabetes_regression/training/train_aml.py b/diabetes_regression/training/train_aml.py
@@ -29,7 +29,7 @@
 import argparse
 import joblib
 import json
-from train import split_data, train_model, get_model_metrics
+from diabetes_regression.training.train import split_data, train_model, get_model_metrics
 
 
 def register_dataset(
@@ -111,7 +111,7 @@ def main():
     print("Getting training parameters")
 
     # Load the training parameters from the parameters file
-    with open("parameters.json") as f:
+    with open("diabetes_regression/parameters.json") as f:
         pars = json.load(f)
     try:
         train_args = pars["training"]

diff --git a/ml_service/pipelines/diabetes_regression_verify_train_pipeline.py b/ml_service/pipelines/diabetes_regression_verify_train_pipeline.py
@@ -15,7 +15,7 @@ def main():
         load_dotenv()
         sources_dir = os.environ.get("SOURCES_DIR_TRAIN")
         if (sources_dir is None):
-            sources_dir = 'diabetes_regression'
+            sources_dir = '.'
         workspace_name = os.environ.get("WORKSPACE_NAME")
         experiment_name = os.environ.get("EXPERIMENT_NAME")
         resource_group = os.environ.get("RESOURCE_GROUP")

diff --git a/ml_service/util/create_scoring_image.py b/ml_service/util/create_scoring_image.py
@@ -27,7 +27,7 @@
 model = Model(ws, name=e.model_name, version=e.model_version)
 sources_dir = e.sources_directory_train
 if (sources_dir is None):
-    sources_dir = 'diabetes_regression'
+    sources_dir = '.'
 score_script = os.path.join(".", sources_dir, e.score_script)
 score_file = os.path.basename(score_script)
 path_to_scoring = os.path.dirname(score_script)

diff --git a/ml_service/util/manage_environment.py b/ml_service/util/manage_environment.py
@@ -1,17 +1,16 @@
-
 import os
 from azureml.core import Workspace, Environment
 from ml_service.util.env_variables import Env
 from azureml.core.runconfig import DEFAULT_CPU_IMAGE, DEFAULT_GPU_IMAGE
 
 
 def get_environment(
-    workspace: Workspace,
-    environment_name: str,
-    conda_dependencies_file: str,
-    create_new: bool = False,
-    enable_docker: bool = None,
-    use_gpu: bool = False
+        workspace: Workspace,
+        environment_name: str,
+        conda_dependencies_file: str,
+        create_new: bool = False,
+        enable_docker: bool = None,
+        use_gpu: bool = False
 ):
     try:
         e = Env()
@@ -24,7 +23,7 @@ def get_environment(
         if restored_environment is None or create_new:
             new_env = Environment.from_conda_specification(
                 environment_name,
-                os.path.join(e.sources_directory_train, conda_dependencies_file),  # NOQA: E501
+                os.path.join(e.sources_directory_train, "diabetes_regression/" + conda_dependencies_file),  # NOQA: E501
             )  # NOQA: E501
             restored_environment = new_env
             if enable_docker is not None: