.

Sushant Divate (sudivate) · Jan 31, 2020 · Nov 28, 2019 · Jan 19, 2020 · Jan 29, 2020 · Jan 29, 2020
commit c8406234257e63fd5a28757f1e5ed3b1afc097b1
diff --git a/diabetes_regression/scoring/inference_config.yml b/diabetes_regression/scoring/inference_config.yml
@@ -1,9 +1,9 @@
 entryScript: score.py
 runtime: python
-condaFile: conda_dependencies.yml
+condaFile: scoring_dependencies.yml
 extraDockerfileSteps:
 schemaFile:
 sourceDirectory:
 enableGpu: False
 baseImage:
-baseImageRegistry:
+baseImageRegistry:
diff --git a/...regression/scoring/conda_dependencies.yml → ...gression/scoring/scoring_dependencies.yml b/...regression/scoring/conda_dependencies.yml → ...gression/scoring/scoring_dependencies.yml
@@ -14,24 +14,23 @@
 # This directive is stored in a comment to preserve the Conda file structure.
 # [AzureMlVersion] = 2
 
-name: project_environment
+name: diabetes_scoring
+
 dependencies:
+
   # The python interpreter version.
-  # Currently Azure ML Workbench only supports 3.5.2 and later.
 - python=3.7.5
+
   # Required by azureml-defaults, installed separately through Conda to
   # get a prebuilt version and not require build tools for the install.
 - psutil=5.6 #latest
 
+- numpy
+- pandas
+- scikit-learn
+
+- pip
 - pip:
-    # Required packages for AzureML execution, history, and data preparation.
-  - azureml-model-management-sdk==1.0.1b6.post1
-  - azureml-sdk==1.0.74
-  - scipy==1.3.1
-  - scikit-learn==0.22
-  - pandas==0.25.3
-  - numpy==1.17.3
-  - joblib==0.14.0
-  - gunicorn==19.9.0 
-  - flask==1.1.1
+  # You must list azureml-defaults as a pip dependency
+  - azureml-defaults>=1.0.45
   - inference-schema[numpy-support]
diff --git a/diabetes_regression/training/training_dependencies.yml b/diabetes_regression/training/training_dependencies.yml
@@ -0,0 +1,17 @@
+name: diabetes_training
+
+dependencies:
+
+  # The python interpreter version.
+- python=3.7.5
+
+- numpy
+- pandas
+- scikit-learn
+- tensorflow
+- keras
+
+- pip
+- pip:
+  - azureml-core
+  - azure-storage-blob
diff --git a/environment_setup/Dockerfile b/environment_setup/Dockerfile
@@ -4,11 +4,16 @@ LABEL org.label-schema.vendor = "Microsoft" \
       org.label-schema.url = "https://hub.docker.com/r/microsoft/mlopspython" \
       org.label-schema.vcs-url = "https://github.com/microsoft/MLOpsPython"
 
-       
+COPY diabetes_regression/training/training_dependencies.yml diabetes_regression/scoring/scoring_dependencies.yml environment_setup/ci_environment.yml /setup/
 
-COPY environment_setup/requirements.txt  /setup/
-	
-RUN apt-get update && apt-get install gcc -y && pip install --upgrade -r /setup/requirements.txt && \ 
-    conda install -c r r-essentials
+RUN pip install conda-merge==0.1.5 && \
+    cd /setup && conda-merge training_dependencies.yml scoring_dependencies.yml ci_environment.yml > conda_merged.yml && \
+    echo "Generated conda environment definition:" && cat conda_merged.yml && \
+    conda env create -f conda_merged.yml
 
-CMD ["python"]
+# activate environment
+ENV PATH /usr/local/envs/ci/bin:$PATH
+RUN /bin/bash -c "source activate ci"
+
+# Install Azure CLI ML extension
+RUN az extension add -n azure-cli-ml
diff --git a/environment_setup/ci_environment.yml b/environment_setup/ci_environment.yml
@@ -0,0 +1,25 @@
+name: ci
+
+dependencies:
+
+- r
+- r-essentials
+- numpy
+- pandas
+- scikit-learn
+
+- pip
+- pip:
+
+  # dependencies shared with diabetes_regression/scoring/scoring_dependencies.yml.
+  # Versions specification must match exactly, or the docker build will fail.
+  - azureml-sdk==1.0.74
+
+  # Additional pip dependencies for the CI environment.
+  - pytest>=5.3
+  - requests>=2.22
+  - numpy>=1.17
+  - python-dotenv>=0.10.3
+  - flake8>=3.7
+  - flake8_formatter_junit_xml
+  - azure-cli>=2.0.76
diff --git a/environment_setup/install_requirements.sh b/environment_setup/install_requirements.sh
@@ -24,8 +24,7 @@
 # ARISING IN ANY WAY OUT OF THE USE OF THE SOFTWARE CODE, EVEN IF ADVISED OF THE
 # POSSIBILITY OF SUCH DAMAGE.
 
-
-python --version
-pip install azure-cli==2.0.46
-pip install --upgrade azureml-sdk[cli]
-pip install -r requirements.txt
+set -eux
+pip install conda-merge==0.1.5
+conda-merge environment_setup/ci_environment.yml diabetes_regression/scoring/scoring_dependencies.yml diabetes_regression/training/training_dependencies.yml > /tmp/conda_merged.yml
+conda env create -f /tmp/conda_merged.yml
diff --git a/environment_setup/requirements.txt b/environment_setup/requirements.txt
diff --git a/ml_service/pipelines/diabetes_regression_build_train_pipeline.py b/ml_service/pipelines/diabetes_regression_build_train_pipeline.py
@@ -31,14 +31,10 @@ def main():
         print("aml_compute:")
         print(aml_compute)
 
-    run_config = RunConfiguration(conda_dependencies=CondaDependencies.create(
-        conda_packages=['numpy', 'pandas',
-                        'scikit-learn', 'tensorflow', 'keras'],
-        pip_packages=['azure', 'azureml-core',
-                      'azure-storage',
-                      'azure-storage-blob',
-                      'azureml-dataprep'])
-    )
+    # Create a reusable run configuration environment
+    conda_deps_file = "diabetes_regression/training/training_dependencies.yml"
+    conda_deps = CondaDependencies(conda_deps_file)
+    run_config = RunConfiguration(conda_dependencies=conda_deps)
     run_config.environment.docker.enabled = True
     config_envvar = {}
     if (e.collection_uri is not None and e.teamproject_name is not None):