timcornwell · May 28, 2018 · May 28, 2018
diff --git a/.gitignore b/.gitignore
@@ -1,4 +1,5 @@
 
+PrivateRules.mak
 /.idea
 /examples/**/*.rst
 /examples/**/*.html

diff --git a/Dockerfile b/Dockerfile
@@ -1,7 +1,7 @@
 # Universal image for running Notebook, Dask pipelines, libs, and lint checkers
 ARG PYTHON=python3.6
 
-FROM ubuntu:17.04
+FROM ubuntu:18.04
 
 ARG PYTHON
 
@@ -17,7 +17,7 @@ ENV DEBIAN_FRONTEND noninteractive
 RUN \
     DEBIAN_FRONTEND=noninteractive apt-get update -y && \
     apt-get install -y locales tzdata python-pip net-tools vim-tiny software-properties-common \
-            python-software-properties build-essential curl wget fonts-liberation ca-certificates && \
+            python3-software-properties build-essential curl wget fonts-liberation ca-certificates && \
     echo "Setting locales  ..." && /usr/sbin/locale-gen en_US.UTF-8 && \
     /usr/sbin/locale-gen en_NZ.UTF-8 && \
     echo "Setting timezone ..." &&  /bin/echo 'Pacific/Auckland' | tee /etc/timezone && DEBIAN_FRONTEND=noninteractive dpkg-reconfigure --frontend noninteractive tzdata && \
@@ -28,18 +28,26 @@ RUN \
     apt-get install -y ${PYTHON}-dev python3-tk flake8 python3-nose \
             virtualenv virtualenvwrapper && \
     apt-get install -y graphviz && \
+    apt-get install -y nodejs npm && \
     apt-get clean -y
 
+# node node is linked to nodejs
+RUN if [ ! -f /usr/bin/node ]; then ln -s /usr/bin/nodejs /usr/bin/node ; fi && \
+    node --version
+
 # sort out pip and python for 3.6
 RUN cd /src; wget https://bootstrap.pypa.io/get-pip.py && ${PYTHON} get-pip.py; \
     rm -rf /root/.cache
 
 RUN if [ "${PYTHON}" = "python3.6" ] ; then ln -s /usr/bin/python3.6 /usr/local/bin/python3 && \
-    rm -f /usr/bin/python3 && ln -s /usr/bin/python3.6 /usr/bin/python3 ; fi
+    rm -f /usr/bin/python3 && ln -s /usr/bin/python3.6 /usr/bin/python3 ; fi && \
+    python3 --version && \
+    pip3 --version
+
 
 # Install Tini
-RUN wget --quiet https://github.com/krallin/tini/releases/download/v0.10.0/tini && \
-    echo "1361527f39190a7338a0b434bd8c88ff7233ce7b9a4876f3315c22fce7eca1b0 *tini" | sha256sum -c - && \
+RUN wget --quiet https://github.com/krallin/tini/releases/download/v0.18.0/tini && \
+    echo "12d20136605531b09a2c2dac02ccee85e1b874eb322ef6baf7561cd93f93c855 *tini" | sha256sum -c - && \
     mv tini /usr/local/bin/tini && \
     chmod +x /usr/local/bin/tini
 
@@ -50,29 +58,48 @@ RUN pip3 install bokeh && pip3 install pytest; pip3 install jupyter_nbextensions
 RUN pip3 install -U pylint; rm -rf /root/.cache
 RUN jupyter contrib nbextension install --system --symlink
 RUN jupyter nbextensions_configurator enable --system
+#RUN pip install jupyterlab
+#RUN jupyter serverextension enable --py jupyterlab --sys-prefix
+#RUN jupyter labextension install @jupyter-widgets/jupyterlab-manager
+#RUN jupyter labextension install jupyterlab_bokeh
 
 # runtime specific environment
 ENV JENKINS_URL 1
 ENV PYTHONPATH /arl
 ENV JUPYTER_PATH /arl/examples/arl
 
+RUN touch "${HOME}/.bash_profile"
+
 # Bundle app source
 ADD ./docker/boot.sh /
-ADD ./arl /arl/arl/
+ADD ./libs /arl/libs/
 ADD ./Makefile /arl/
 ADD ./examples /arl/examples/
 ADD ./tests /arl/tests/
+ADD ./setup.py ./README.md /arl/
+ADD ./data_models /arl/data_models/
+ADD ./processing_components /arl/processing_components/
+ADD ./util /arl/util/
+ADD ./workflows /arl/workflows/
+
+# run setup
+RUN cd /arl && ${PYTHON} setup.py build && ${PYTHON} setup.py install
 
 # create space for libs
 RUN mkdir -p /arl/test_data /arl/test_results && \
     chmod 777 /arl /arl/test_data /arl/test_results && \
     chmod -R a+w /arl
 
+COPY --chown="1000:100" ./docker/jupyter_notebook_config.py "${HOME}/.jupyter/"
+COPY ./docker/notebook.sh /usr/local/bin/
+COPY ./docker/start-dask-scheduler.sh /usr/local/bin/
+COPY ./docker/start-dask-worker.sh /usr/local/bin
+
 # We share in the arl data here
 VOLUME ["/arl/data", "/arl/tmp"]
 
 # Expose Jupyter and Bokeh ports
-EXPOSE  8888 8787
+EXPOSE  8888 8786 8787 8788 8789
 
 # Setup the entrypoint or environment
 ENTRYPOINT ["tini", "--"]

diff --git a/Makefile b/Makefile
@@ -6,20 +6,40 @@ FLAKE ?= flake8
 NAME = arl
 IMG = $(NAME)_img
 TAG = latest
+DOCKER_IMAGE = $(IMG):$(TAG)
 DOCKERFILE = Dockerfile
 DOCKER = docker
-DOCKER_REPO ?= localhost:5000
-
-
-all: clean nosetests
+DOCKER_REPO ?= ""
+DOCKER_USER ?= ""
+DOCKER_PASSWORD ?= ""
+WORKER_MEM ?= 512Mi
+WORKER_CPU ?= 500m
+WORKER_REPLICAS ?= 1
+WORKER_ARL_DATA ?= /arl/data
+CURRENT_DIR = $(shell pwd)
+JUPYTER_PASSWORD ?= changeme
+CONTAINER_EXISTS = $(shell $(DOCKER) ps -aqf ancestor=itsthenetwork/nfs-server-alpine)
+SERVER_DEVICE ?= $(shell ip link | grep BROADCAST | head -1 | awk '{print $$2}' | sed 's/://')
+NFS_SERVER ?= "127.0.0.1"
+
+# define overides for above variables in here
+-include PrivateRules.mak
+
+checkvars:
+	@echo "Image: $(DOCKER_IMAGE)"
+	@echo "Repo: $(DOCKER_REPO)"
+	@echo "Host net device: $(SERVER_DEVICE)"
+	@echo "Nfs: $(NFS_SERVER)"
+
+all: clean build nosetests
 
 clean:
 	$(PYTHON) setup.py clean
 	rm -rf dist
 
 in: inplace # just a shortcut
 inplace:
-	$(PYTHON) setup.py install >/dev/null 2>&1 || (echo "'$(PYTHON) setup.py install' failed."; exit -1)
+	$(PYTHON) setup.py build >/dev/null 2>&1 && $(PYTHON) setup.py install >/dev/null 2>&1 || (echo "'$(PYTHON) setup.py install' failed."; exit -1)
 
 build: in
 
@@ -39,7 +59,7 @@ nosetests: cleantests
 
 nosetests-coverage: inplace cleantests
 	rm -rf coverage .coverage
-	$(NOSETESTS) -s -v --with-coverage arl
+	$(NOSETESTS) -s -v --with-coverage libs
 
 trailing-spaces:
 	find libs -name "*.py" -exec perl -pi -e 's/[ \t]*$$//' {} \;
@@ -52,12 +72,12 @@ docs: inplace
 
 code-flake:
 	# flake8 ignore long lines and trailing whitespace
-	$(FLAKE) --ignore=E501,W293,F401 --builtins=ModuleNotFoundError arl
+	$(FLAKE) --ignore=E501,W293,F401 --builtins=ModuleNotFoundError libs
 
 code-lint:
 	$(PYLINT) --extension-pkg-whitelist=numpy \
 	  --ignored-classes=astropy.units,astropy.constants,HDUList \
-	  -E arl/ tests/
+	  -E libs/ tests/
 
 code-analysis: code-flake code-lint
 
@@ -73,11 +93,55 @@ notebook:
 	jupyter notebook --no-browser --ip=$${IP} --port=8888 examples/arl/
 
 docker_build:
-	$(DOCKER) build -t $(IMG) -f $(DOCKERFILE) --build-arg PYTHON=$(PYTHON) .
+	$(DOCKER) build -t $(DOCKER_IMAGE) -f $(DOCKERFILE) --build-arg PYTHON=$(PYTHON) .
 
 docker_push: docker_build
-	docker tag $(IMG) $(DOCKER_REPO)/$(IMG):$(TAG)
-	docker push $(DOCKER_REPO)/$(IMG):$(TAG)
+	docker tag $(DOCKER_IMAGE) $(DOCKER_REPO)$(DOCKER_IMAGE)
+	docker push $(DOCKER_REPO)$(DOCKER_IMAGE)
+
+docker_nfs_arl_data:
+ifneq "$(strip $(CONTAINER_EXISTS))" ""
+	$(DOCKER) rm -f $(CONTAINER_EXISTS)
+endif
+	docker run -d --name nfs --privileged -p 2049:2049 \
+	-v $(CURRENT_DIR)/:/arl \
+	-e SHARED_DIRECTORY=/arl itsthenetwork/nfs-server-alpine:latest
+
+k8s_deploy_scheduler:
+	DOCKER_IMAGE=$(DOCKER_REPO)$(DOCKER_IMAGE) \
+	 envsubst < k8s/k8s-dask-scheduler-deployment.yml | kubectl apply -f -
+
+k8s_delete_scheduler:
+	DOCKER_IMAGE=$(DOCKER_REPO)$(DOCKER_IMAGE) \
+	 envsubst < k8s/k8s-dask-scheduler-deployment.yml | kubectl delete -f - || true
+
+k8s_deploy_worker:
+	DOCKER_IMAGE=$(DOCKER_REPO)$(DOCKER_IMAGE) \
+	WORKER_MEM=$(WORKER_MEM) WORKER_CPU=$(WORKER_CPU) \
+	WORKER_REPLICAS=$(WORKER_REPLICAS) \
+	WORKER_ARL_DATA=$(WORKER_ARL_DATA) \
+	NFS_SERVER=$(NFS_SERVER) \
+	 envsubst < k8s/k8s-dask-worker-deployment.yml | kubectl apply -f -
+
+k8s_delete_worker:
+	DOCKER_IMAGE=$(DOCKER_REPO)$(DOCKER_IMAGE) \
+	WORKER_MEM=$(WORKER_MEM) WORKER_CPU=$(WORKER_CPU) \
+	WORKER_REPLICAS=$(WORKER_REPLICAS) \
+	WORKER_ARL_DATA=$(WORKER_ARL_DATA) \
+	NFS_SERVER=$(NFS_SERVER) \
+	 envsubst < k8s/k8s-dask-worker-deployment.yml | kubectl delete -f - || true
+
+k8s_deploy_notebook:
+	DOCKER_IMAGE=$(DOCKER_REPO)$(DOCKER_IMAGE) \
+	WORKER_ARL_DATA=$(WORKER_ARL_DATA) \
+	NFS_SERVER=$(NFS_SERVER) \
+	 envsubst < k8s/k8s-dask-notebook-deployment.yml | kubectl apply -f -
+
+k8s_delete_notebook:
+	DOCKER_IMAGE=$(DOCKER_REPO)$(DOCKER_IMAGE) \
+	WORKER_ARL_DATA=$(WORKER_ARL_DATA) \
+	NFS_SERVER=$(NFS_SERVER) \
+	 envsubst < k8s/k8s-dask-notebook-deployment.yml | kubectl delete -f - || true
 
 docker_notebook: docker_build
 	CTNR=`$(DOCKER) ps -q -f name=$(NAME)_notebook` && \
@@ -86,10 +150,14 @@ docker_notebook: docker_build
 	IP=`ip a show $${DEVICE} | grep ' inet ' | awk '{print $$2}' | sed 's/\/.*//'` && \
 	echo "Launching at IP: $${IP}" && \
 	$(DOCKER) run --name $(NAME)_notebook --hostname $(NAME)_notebook --volume $$(pwd)/data:/arl/data -e IP=$${IP} \
-            --net=host -p 8888:8888 -p 8787:8787 -p 8788:8788 -p 8789:8789 -d $(IMG)
+            --net=host -p 8888:8888 -p 8787:8787 -p 8788:8788 -p 8789:8789 -d $(DOCKER_IMAGE)
 	sleep 3
 	$(DOCKER) logs $(NAME)_notebook
 
+k8s_deploy: k8s_deploy_scheduler k8s_deploy_worker k8s_deploy_notebook  
+
+k8s_delete: k8s_delete_notebook k8s_delete_worker k8s_delete_scheduler
+
 docker_tests: docker_build cleantests
 	rm -f predict_facet_timeslice_graph_wprojection.png pipelines-timings_*.csv
 	CTNR=`$(DOCKER) ps -q -f name=$(NAME)_tests` && \
@@ -98,7 +166,7 @@ docker_tests: docker_build cleantests
 					-v /etc/passwd:/etc/passwd:ro --user=$$(id -u) \
 					-v $${HOME}:$${HOME} -w $${HOME} \
 					-e HOME=$${HOME} \
-		            --net=host -ti $(IMG) /bin/sh -c "cd /arl && make nosetests"
+		            --net=host -ti $(DOCKER_IMAGE) /bin/sh -c "cd /arl && make nosetests"
 
 docker_pytest: docker_build cleantests
 	CTNR=`$(DOCKER) ps -q -f name=$(NAME)_tests` && \
@@ -107,7 +175,7 @@ docker_pytest: docker_build cleantests
 					-v /etc/passwd:/etc/passwd:ro --user=$$(id -u) \
 					-v $${HOME}:$${HOME} -w $${HOME} \
 					-e HOME=$${HOME} \
-			    --net=host -ti $(IMG) /bin/sh -c "cd /arl && make pytest"
+			    --net=host -ti $(DOCKER_IMAGE) /bin/sh -c "cd /arl && make pytest"
 
 docker_lint: docker_build
 	CTNR=`$(DOCKER) ps -q -f name=$(NAME)_lint` && \
@@ -116,7 +184,7 @@ docker_lint: docker_build
 					-v /etc/passwd:/etc/passwd:ro --user=$$(id -u) \
 					-v $${HOME}:$${HOME} -w $${HOME} \
 					-e HOME=$${HOME} \
-		            --net=host -ti $(IMG) /bin/sh -c "cd /arl && make code-analysis"
+		            --net=host -ti $(DOCKER_IMAGE) /bin/sh -c "cd /arl && make code-analysis"
 
 launch_dask:
 	cd tools && ansible-playbook -i ./inventory ./docker.yml
diff --git a/docker/boot.sh b/docker/boot.sh
@@ -1,5 +1,13 @@
-#!/bin/sh
+#!/usr/bin/env bash
+
+source "${HOME}/.bash_profile"
 
 # launch the notebook - the IP address to listen on is passed in via env-var IP
 IP=${IP:-0.0.0.0}
-jupyter notebook --allow-root --no-browser --ip=${IP} --port=8888 /arl/examples/arl/
+NOTEBOOK_PORT=${NOTEBOOK_PORT:-8888}
+export JUPYTER_PASSWORD=${JUPYTER_PASSWORD:-changeme}
+jupyter notebook --allow-root --no-browser --ip=${IP} --port=${NOTEBOOK_PORT} /arl
+#jupyter lab --allow-root --no-browser --ip=${IP} --port=${PORT} "$@" /arl/examples/arl/
+
+# source activate dask-distributed
+#jupyter lab --allow-root "$@"
diff --git a/docker/jupyter_notebook_config.py b/docker/jupyter_notebook_config.py
@@ -0,0 +1,90 @@
+import errno
+import os
+import stat
+import subprocess
+
+
+from jupyter_core.paths import jupyter_data_dir
+from notebook.auth import passwd
+
+# Setup the Notebook to listen on all interfaces on port 8888 by default
+c.NotebookApp.ip = '*'
+c.NotebookApp.port = 8888
+c.NotebookApp.open_browser = False
+
+# Configure Networking while running under Marathon:
+if 'MARATHON_APP_ID' in os.environ:
+    if 'PORT_JUPYTER' in os.environ:
+        c.NotebookApp.port = int(os.environ['PORT_JUPYTER'])
+
+    # Set the Access-Control-Allow-Origin header
+    c.NotebookApp.allow_origin = '*'
+
+    # Set Jupyter Notebook Server password to 'jupyter-<Marathon-App-Prefix>'
+    # e.g., Marathon App ID '/foo/bar/app' maps to password: 'jupyter-foo-bar'
+    MARATHON_APP_PREFIX = \
+        '-'.join(os.environ['MARATHON_APP_ID'].split('/')[:-1])
+    c.NotebookApp.password = passwd('jupyter{}'.format(MARATHON_APP_PREFIX))
+
+    # Allow CORS and TLS from behind Marathon-LB/HAProxy
+    # Trust X-Scheme/X-Forwarded-Proto and X-Real-Ip/X-Forwarded-For
+    # Necessary if the proxy handles SSL
+    if 'MARATHON_APP_LABEL_HAPROXY_GROUP' in os.environ:
+        c.NotebookApp.trust_xheaders = True
+
+    if 'MARATHON_APP_LABEL_HAPROXY_0_VHOST' in os.environ:
+        c.NotebookApp.allow_origin = \
+            'http://{}'.format(
+                os.environ['MARATHON_APP_LABEL_HAPROXY_0_VHOST']
+            )
+
+    if 'MARATHON_APP_LABEL_HAPROXY_0_REDIRECT_TO_HTTPS' in os.environ:
+        c.NotebookApp.allow_origin = \
+            'https://{}'.format(
+                os.environ['MARATHON_APP_LABEL_HAPROXY_0_VHOST']
+            )
+
+    # Set the Jupyter Notebook server base URL to the HAPROXY_PATH specified
+    if 'MARATHON_APP_LABEL_HAPROXY_0_PATH' in os.environ:
+        c.NotebookApp.base_url = \
+            os.environ['MARATHON_APP_LABEL_HAPROXY_0_PATH']
+
+    # Setup TLS
+    if 'USE_HTTPS' in os.environ:
+        SCHEDULER_TLS_CERT = os.environ.get('TLS_CERT_PATH', '/'.join(
+            [os.environ['MESOS_SANDBOX'],
+             '.ssl',
+             'scheduler.crt']))
+        c.NotebookApp.certfile = SCHEDULER_TLS_CERT
+        SCHEDULER_TLS_KEY = os.environ.get('TLS_KEY_PATH', '/'.join(
+            [os.environ['MESOS_SANDBOX'],
+             '.ssl',
+             'scheduler.key']))
+        c.NotebookApp.keyfile = SCHEDULER_TLS_KEY
+
+# Set a certificate if USE_HTTPS is set to any value
+PEM_FILE = os.path.join(jupyter_data_dir(), 'notebook.pem')
+if 'USE_HTTPS' in os.environ:
+    if not os.path.isfile(PEM_FILE):
+        # Ensure PEM_FILE directory exists
+        DIR_NAME = os.path.dirname(PEM_FILE)
+        try:
+            os.makedirs(DIR_NAME)
+        except OSError as exc:  # Python >2.5
+            if exc.errno == errno.EEXIST and os.path.isdir(DIR_NAME):
+                pass
+            else:
+                raise
+        # Generate a certificate if one doesn't exist on disk
+        subprocess.check_call(['openssl', 'req', '-new', '-newkey', 'rsa:2048',
+                               '-days', '365', '-nodes', '-x509', '-subj',
+                               '/C=XX/ST=XX/L=XX/O=generated/CN=generated',
+                               '-keyout', PEM_FILE, '-out', PEM_FILE])
+        # Restrict access to PEM_FILE
+        os.chmod(PEM_FILE, stat.S_IRUSR | stat.S_IWUSR)
+    c.NotebookApp.certfile = PEM_FILE
+
+# Set a password if JUPYTER_PASSWORD is set
+if 'JUPYTER_PASSWORD' in os.environ:
+    c.NotebookApp.password = passwd(os.environ['JUPYTER_PASSWORD'])
+    del os.environ['JUPYTER_PASSWORD']