Skip to content

Commit b67dac0

Browse files
author
Piers Harding
committed
Add kubernetes support
* upgrade docker build to Ubuntu 18.04 * add notebook, scheduler, and work support to Dockerfile * add k8s deployment descriptors * instructions for running on k8s and minikube
1 parent 7fa4cf3 commit b67dac0

13 files changed

Lines changed: 1410 additions & 24 deletions

‎.gitignore‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,5 @@
11

2+
PrivateRules.mak
23
/.idea
34
/examples/**/*.rst
45
/examples/**/*.html

‎Dockerfile‎

Lines changed: 34 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -1,7 +1,7 @@
11
# Universal image for running Notebook, Dask pipelines, libs, and lint checkers
22
ARG PYTHON=python3.6
33

4-
FROM ubuntu:17.04
4+
FROM ubuntu:18.04
55

66
ARG PYTHON
77

@@ -17,7 +17,7 @@ ENV DEBIAN_FRONTEND noninteractive
1717
RUN \
1818
DEBIAN_FRONTEND=noninteractive apt-get update -y && \
1919
apt-get install -y locales tzdata python-pip net-tools vim-tiny software-properties-common \
20-
python-software-properties build-essential curl wget fonts-liberation ca-certificates && \
20+
python3-software-properties build-essential curl wget fonts-liberation ca-certificates && \
2121
echo "Setting locales ..." && /usr/sbin/locale-gen en_US.UTF-8 && \
2222
/usr/sbin/locale-gen en_NZ.UTF-8 && \
2323
echo "Setting timezone ..." && /bin/echo 'Pacific/Auckland' | tee /etc/timezone && DEBIAN_FRONTEND=noninteractive dpkg-reconfigure --frontend noninteractive tzdata && \
@@ -28,18 +28,26 @@ RUN \
2828
apt-get install -y ${PYTHON}-dev python3-tk flake8 python3-nose \
2929
virtualenv virtualenvwrapper && \
3030
apt-get install -y graphviz && \
31+
apt-get install -y nodejs npm && \
3132
apt-get clean -y
3233

34+
# node node is linked to nodejs
35+
RUN if [ ! -f /usr/bin/node ]; then ln -s /usr/bin/nodejs /usr/bin/node ; fi && \
36+
node --version
37+
3338
# sort out pip and python for 3.6
3439
RUN cd /src; wget https://bootstrap.pypa.io/get-pip.py && ${PYTHON} get-pip.py; \
3540
rm -rf /root/.cache
3641

3742
RUN if [ "${PYTHON}" = "python3.6" ] ; then ln -s /usr/bin/python3.6 /usr/local/bin/python3 && \
38-
rm -f /usr/bin/python3 && ln -s /usr/bin/python3.6 /usr/bin/python3 ; fi
43+
rm -f /usr/bin/python3 && ln -s /usr/bin/python3.6 /usr/bin/python3 ; fi && \
44+
python3 --version && \
45+
pip3 --version
46+
3947

4048
# Install Tini
41-
RUN wget --quiet https://github.com/krallin/tini/releases/download/v0.10.0/tini && \
42-
echo "1361527f39190a7338a0b434bd8c88ff7233ce7b9a4876f3315c22fce7eca1b0 *tini" | sha256sum -c - && \
49+
RUN wget --quiet https://github.com/krallin/tini/releases/download/v0.18.0/tini && \
50+
echo "12d20136605531b09a2c2dac02ccee85e1b874eb322ef6baf7561cd93f93c855 *tini" | sha256sum -c - && \
4351
mv tini /usr/local/bin/tini && \
4452
chmod +x /usr/local/bin/tini
4553

@@ -50,29 +58,48 @@ RUN pip3 install bokeh && pip3 install pytest; pip3 install jupyter_nbextensions
5058
RUN pip3 install -U pylint; rm -rf /root/.cache
5159
RUN jupyter contrib nbextension install --system --symlink
5260
RUN jupyter nbextensions_configurator enable --system
61+
#RUN pip install jupyterlab
62+
#RUN jupyter serverextension enable --py jupyterlab --sys-prefix
63+
#RUN jupyter labextension install @jupyter-widgets/jupyterlab-manager
64+
#RUN jupyter labextension install jupyterlab_bokeh
5365

5466
# runtime specific environment
5567
ENV JENKINS_URL 1
5668
ENV PYTHONPATH /arl
5769
ENV JUPYTER_PATH /arl/examples/arl
5870

71+
RUN touch "${HOME}/.bash_profile"
72+
5973
# Bundle app source
6074
ADD ./docker/boot.sh /
61-
ADD ./arl /arl/arl/
75+
ADD ./libs /arl/libs/
6276
ADD ./Makefile /arl/
6377
ADD ./examples /arl/examples/
6478
ADD ./tests /arl/tests/
79+
ADD ./setup.py ./README.md /arl/
80+
ADD ./data_models /arl/data_models/
81+
ADD ./processing_components /arl/processing_components/
82+
ADD ./util /arl/util/
83+
ADD ./workflows /arl/workflows/
84+
85+
# run setup
86+
RUN cd /arl && ${PYTHON} setup.py build && ${PYTHON} setup.py install
6587

6688
# create space for libs
6789
RUN mkdir -p /arl/test_data /arl/test_results && \
6890
chmod 777 /arl /arl/test_data /arl/test_results && \
6991
chmod -R a+w /arl
7092

93+
COPY --chown="1000:100" ./docker/jupyter_notebook_config.py "${HOME}/.jupyter/"
94+
COPY ./docker/notebook.sh /usr/local/bin/
95+
COPY ./docker/start-dask-scheduler.sh /usr/local/bin/
96+
COPY ./docker/start-dask-worker.sh /usr/local/bin
97+
7198
# We share in the arl data here
7299
VOLUME ["/arl/data", "/arl/tmp"]
73100

74101
# Expose Jupyter and Bokeh ports
75-
EXPOSE 8888 8787
102+
EXPOSE 8888 8786 8787 8788 8789
76103

77104
# Setup the entrypoint or environment
78105
ENTRYPOINT ["tini", "--"]

‎Makefile‎

Lines changed: 83 additions & 15 deletions
Original file line numberDiff line numberDiff line change
@@ -6,20 +6,40 @@ FLAKE ?= flake8
66
NAME = arl
77
IMG = $(NAME)_img
88
TAG = latest
9+
DOCKER_IMAGE = $(IMG):$(TAG)
910
DOCKERFILE = Dockerfile
1011
DOCKER = docker
11-
DOCKER_REPO ?= localhost:5000
12-
13-
14-
all: clean nosetests
12+
DOCKER_REPO ?= ""
13+
DOCKER_USER ?= ""
14+
DOCKER_PASSWORD ?= ""
15+
WORKER_MEM ?= 512Mi
16+
WORKER_CPU ?= 500m
17+
WORKER_REPLICAS ?= 1
18+
WORKER_ARL_DATA ?= /arl/data
19+
CURRENT_DIR = $(shell pwd)
20+
JUPYTER_PASSWORD ?= changeme
21+
CONTAINER_EXISTS = $(shell $(DOCKER) ps -aqf ancestor=itsthenetwork/nfs-server-alpine)
22+
SERVER_DEVICE ?= $(shell ip link | grep BROADCAST | head -1 | awk '{print $$2}' | sed 's/://')
23+
NFS_SERVER ?= "127.0.0.1"
24+
25+
# define overides for above variables in here
26+
-include PrivateRules.mak
27+
28+
checkvars:
29+
@echo "Image: $(DOCKER_IMAGE)"
30+
@echo "Repo: $(DOCKER_REPO)"
31+
@echo "Host net device: $(SERVER_DEVICE)"
32+
@echo "Nfs: $(NFS_SERVER)"
33+
34+
all: clean build nosetests
1535

1636
clean:
1737
$(PYTHON) setup.py clean
1838
rm -rf dist
1939

2040
in: inplace # just a shortcut
2141
inplace:
22-
$(PYTHON) setup.py install >/dev/null 2>&1 || (echo "'$(PYTHON) setup.py install' failed."; exit -1)
42+
$(PYTHON) setup.py build >/dev/null 2>&1 && $(PYTHON) setup.py install >/dev/null 2>&1 || (echo "'$(PYTHON) setup.py install' failed."; exit -1)
2343

2444
build: in
2545

@@ -39,7 +59,7 @@ nosetests: cleantests
3959

4060
nosetests-coverage: inplace cleantests
4161
rm -rf coverage .coverage
42-
$(NOSETESTS) -s -v --with-coverage arl
62+
$(NOSETESTS) -s -v --with-coverage libs
4363

4464
trailing-spaces:
4565
find libs -name "*.py" -exec perl -pi -e 's/[ \t]*$$//' {} \;
@@ -52,12 +72,12 @@ docs: inplace
5272

5373
code-flake:
5474
# flake8 ignore long lines and trailing whitespace
55-
$(FLAKE) --ignore=E501,W293,F401 --builtins=ModuleNotFoundError arl
75+
$(FLAKE) --ignore=E501,W293,F401 --builtins=ModuleNotFoundError libs
5676

5777
code-lint:
5878
$(PYLINT) --extension-pkg-whitelist=numpy \
5979
--ignored-classes=astropy.units,astropy.constants,HDUList \
60-
-E arl/ tests/
80+
-E libs/ tests/
6181

6282
code-analysis: code-flake code-lint
6383

@@ -73,11 +93,55 @@ notebook:
7393
jupyter notebook --no-browser --ip=$${IP} --port=8888 examples/arl/
7494

7595
docker_build:
76-
$(DOCKER) build -t $(IMG) -f $(DOCKERFILE) --build-arg PYTHON=$(PYTHON) .
96+
$(DOCKER) build -t $(DOCKER_IMAGE) -f $(DOCKERFILE) --build-arg PYTHON=$(PYTHON) .
7797

7898
docker_push: docker_build
79-
docker tag $(IMG) $(DOCKER_REPO)/$(IMG):$(TAG)
80-
docker push $(DOCKER_REPO)/$(IMG):$(TAG)
99+
docker tag $(DOCKER_IMAGE) $(DOCKER_REPO)$(DOCKER_IMAGE)
100+
docker push $(DOCKER_REPO)$(DOCKER_IMAGE)
101+
102+
docker_nfs_arl_data:
103+
ifneq "$(strip $(CONTAINER_EXISTS))" ""
104+
$(DOCKER) rm -f $(CONTAINER_EXISTS)
105+
endif
106+
docker run -d --name nfs --privileged -p 2049:2049 \
107+
-v $(CURRENT_DIR)/:/arl \
108+
-e SHARED_DIRECTORY=/arl itsthenetwork/nfs-server-alpine:latest
109+
110+
k8s_deploy_scheduler:
111+
DOCKER_IMAGE=$(DOCKER_REPO)$(DOCKER_IMAGE) \
112+
envsubst < k8s/k8s-dask-scheduler-deployment.yml | kubectl apply -f -
113+
114+
k8s_delete_scheduler:
115+
DOCKER_IMAGE=$(DOCKER_REPO)$(DOCKER_IMAGE) \
116+
envsubst < k8s/k8s-dask-scheduler-deployment.yml | kubectl delete -f - || true
117+
118+
k8s_deploy_worker:
119+
DOCKER_IMAGE=$(DOCKER_REPO)$(DOCKER_IMAGE) \
120+
WORKER_MEM=$(WORKER_MEM) WORKER_CPU=$(WORKER_CPU) \
121+
WORKER_REPLICAS=$(WORKER_REPLICAS) \
122+
WORKER_ARL_DATA=$(WORKER_ARL_DATA) \
123+
NFS_SERVER=$(NFS_SERVER) \
124+
envsubst < k8s/k8s-dask-worker-deployment.yml | kubectl apply -f -
125+
126+
k8s_delete_worker:
127+
DOCKER_IMAGE=$(DOCKER_REPO)$(DOCKER_IMAGE) \
128+
WORKER_MEM=$(WORKER_MEM) WORKER_CPU=$(WORKER_CPU) \
129+
WORKER_REPLICAS=$(WORKER_REPLICAS) \
130+
WORKER_ARL_DATA=$(WORKER_ARL_DATA) \
131+
NFS_SERVER=$(NFS_SERVER) \
132+
envsubst < k8s/k8s-dask-worker-deployment.yml | kubectl delete -f - || true
133+
134+
k8s_deploy_notebook:
135+
DOCKER_IMAGE=$(DOCKER_REPO)$(DOCKER_IMAGE) \
136+
WORKER_ARL_DATA=$(WORKER_ARL_DATA) \
137+
NFS_SERVER=$(NFS_SERVER) \
138+
envsubst < k8s/k8s-dask-notebook-deployment.yml | kubectl apply -f -
139+
140+
k8s_delete_notebook:
141+
DOCKER_IMAGE=$(DOCKER_REPO)$(DOCKER_IMAGE) \
142+
WORKER_ARL_DATA=$(WORKER_ARL_DATA) \
143+
NFS_SERVER=$(NFS_SERVER) \
144+
envsubst < k8s/k8s-dask-notebook-deployment.yml | kubectl delete -f - || true
81145

82146
docker_notebook: docker_build
83147
CTNR=`$(DOCKER) ps -q -f name=$(NAME)_notebook` && \
@@ -86,10 +150,14 @@ docker_notebook: docker_build
86150
IP=`ip a show $${DEVICE} | grep ' inet ' | awk '{print $$2}' | sed 's/\/.*//'` && \
87151
echo "Launching at IP: $${IP}" && \
88152
$(DOCKER) run --name $(NAME)_notebook --hostname $(NAME)_notebook --volume $$(pwd)/data:/arl/data -e IP=$${IP} \
89-
--net=host -p 8888:8888 -p 8787:8787 -p 8788:8788 -p 8789:8789 -d $(IMG)
153+
--net=host -p 8888:8888 -p 8787:8787 -p 8788:8788 -p 8789:8789 -d $(DOCKER_IMAGE)
90154
sleep 3
91155
$(DOCKER) logs $(NAME)_notebook
92156

157+
k8s_deploy: k8s_deploy_scheduler k8s_deploy_worker k8s_deploy_notebook
158+
159+
k8s_delete: k8s_delete_notebook k8s_delete_worker k8s_delete_scheduler
160+
93161
docker_tests: docker_build cleantests
94162
rm -f predict_facet_timeslice_graph_wprojection.png pipelines-timings_*.csv
95163
CTNR=`$(DOCKER) ps -q -f name=$(NAME)_tests` && \
@@ -98,7 +166,7 @@ docker_tests: docker_build cleantests
98166
-v /etc/passwd:/etc/passwd:ro --user=$$(id -u) \
99167
-v $${HOME}:$${HOME} -w $${HOME} \
100168
-e HOME=$${HOME} \
101-
--net=host -ti $(IMG) /bin/sh -c "cd /arl && make nosetests"
169+
--net=host -ti $(DOCKER_IMAGE) /bin/sh -c "cd /arl && make nosetests"
102170

103171
docker_pytest: docker_build cleantests
104172
CTNR=`$(DOCKER) ps -q -f name=$(NAME)_tests` && \
@@ -107,7 +175,7 @@ docker_pytest: docker_build cleantests
107175
-v /etc/passwd:/etc/passwd:ro --user=$$(id -u) \
108176
-v $${HOME}:$${HOME} -w $${HOME} \
109177
-e HOME=$${HOME} \
110-
--net=host -ti $(IMG) /bin/sh -c "cd /arl && make pytest"
178+
--net=host -ti $(DOCKER_IMAGE) /bin/sh -c "cd /arl && make pytest"
111179

112180
docker_lint: docker_build
113181
CTNR=`$(DOCKER) ps -q -f name=$(NAME)_lint` && \
@@ -116,7 +184,7 @@ docker_lint: docker_build
116184
-v /etc/passwd:/etc/passwd:ro --user=$$(id -u) \
117185
-v $${HOME}:$${HOME} -w $${HOME} \
118186
-e HOME=$${HOME} \
119-
--net=host -ti $(IMG) /bin/sh -c "cd /arl && make code-analysis"
187+
--net=host -ti $(DOCKER_IMAGE) /bin/sh -c "cd /arl && make code-analysis"
120188

121189
launch_dask:
122190
cd tools && ansible-playbook -i ./inventory ./docker.yml

‎docker/boot.sh‎

Lines changed: 10 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,13 @@
1-
#!/bin/sh
1+
#!/usr/bin/env bash
2+
3+
source "${HOME}/.bash_profile"
24

35
# launch the notebook - the IP address to listen on is passed in via env-var IP
46
IP=${IP:-0.0.0.0}
5-
jupyter notebook --allow-root --no-browser --ip=${IP} --port=8888 /arl/examples/arl/
7+
NOTEBOOK_PORT=${NOTEBOOK_PORT:-8888}
8+
export JUPYTER_PASSWORD=${JUPYTER_PASSWORD:-changeme}
9+
jupyter notebook --allow-root --no-browser --ip=${IP} --port=${NOTEBOOK_PORT} /arl
10+
#jupyter lab --allow-root --no-browser --ip=${IP} --port=${PORT} "$@" /arl/examples/arl/
11+
12+
# source activate dask-distributed
13+
#jupyter lab --allow-root "$@"

‎docker/jupyter_notebook_config.py‎

Lines changed: 90 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,90 @@
1+
import errno
2+
import os
3+
import stat
4+
import subprocess
5+
6+
7+
from jupyter_core.paths import jupyter_data_dir
8+
from notebook.auth import passwd
9+
10+
# Setup the Notebook to listen on all interfaces on port 8888 by default
11+
c.NotebookApp.ip = '*'
12+
c.NotebookApp.port = 8888
13+
c.NotebookApp.open_browser = False
14+
15+
# Configure Networking while running under Marathon:
16+
if 'MARATHON_APP_ID' in os.environ:
17+
if 'PORT_JUPYTER' in os.environ:
18+
c.NotebookApp.port = int(os.environ['PORT_JUPYTER'])
19+
20+
# Set the Access-Control-Allow-Origin header
21+
c.NotebookApp.allow_origin = '*'
22+
23+
# Set Jupyter Notebook Server password to 'jupyter-<Marathon-App-Prefix>'
24+
# e.g., Marathon App ID '/foo/bar/app' maps to password: 'jupyter-foo-bar'
25+
MARATHON_APP_PREFIX = \
26+
'-'.join(os.environ['MARATHON_APP_ID'].split('/')[:-1])
27+
c.NotebookApp.password = passwd('jupyter{}'.format(MARATHON_APP_PREFIX))
28+
29+
# Allow CORS and TLS from behind Marathon-LB/HAProxy
30+
# Trust X-Scheme/X-Forwarded-Proto and X-Real-Ip/X-Forwarded-For
31+
# Necessary if the proxy handles SSL
32+
if 'MARATHON_APP_LABEL_HAPROXY_GROUP' in os.environ:
33+
c.NotebookApp.trust_xheaders = True
34+
35+
if 'MARATHON_APP_LABEL_HAPROXY_0_VHOST' in os.environ:
36+
c.NotebookApp.allow_origin = \
37+
'http://{}'.format(
38+
os.environ['MARATHON_APP_LABEL_HAPROXY_0_VHOST']
39+
)
40+
41+
if 'MARATHON_APP_LABEL_HAPROXY_0_REDIRECT_TO_HTTPS' in os.environ:
42+
c.NotebookApp.allow_origin = \
43+
'https://{}'.format(
44+
os.environ['MARATHON_APP_LABEL_HAPROXY_0_VHOST']
45+
)
46+
47+
# Set the Jupyter Notebook server base URL to the HAPROXY_PATH specified
48+
if 'MARATHON_APP_LABEL_HAPROXY_0_PATH' in os.environ:
49+
c.NotebookApp.base_url = \
50+
os.environ['MARATHON_APP_LABEL_HAPROXY_0_PATH']
51+
52+
# Setup TLS
53+
if 'USE_HTTPS' in os.environ:
54+
SCHEDULER_TLS_CERT = os.environ.get('TLS_CERT_PATH', '/'.join(
55+
[os.environ['MESOS_SANDBOX'],
56+
'.ssl',
57+
'scheduler.crt']))
58+
c.NotebookApp.certfile = SCHEDULER_TLS_CERT
59+
SCHEDULER_TLS_KEY = os.environ.get('TLS_KEY_PATH', '/'.join(
60+
[os.environ['MESOS_SANDBOX'],
61+
'.ssl',
62+
'scheduler.key']))
63+
c.NotebookApp.keyfile = SCHEDULER_TLS_KEY
64+
65+
# Set a certificate if USE_HTTPS is set to any value
66+
PEM_FILE = os.path.join(jupyter_data_dir(), 'notebook.pem')
67+
if 'USE_HTTPS' in os.environ:
68+
if not os.path.isfile(PEM_FILE):
69+
# Ensure PEM_FILE directory exists
70+
DIR_NAME = os.path.dirname(PEM_FILE)
71+
try:
72+
os.makedirs(DIR_NAME)
73+
except OSError as exc: # Python >2.5
74+
if exc.errno == errno.EEXIST and os.path.isdir(DIR_NAME):
75+
pass
76+
else:
77+
raise
78+
# Generate a certificate if one doesn't exist on disk
79+
subprocess.check_call(['openssl', 'req', '-new', '-newkey', 'rsa:2048',
80+
'-days', '365', '-nodes', '-x509', '-subj',
81+
'/C=XX/ST=XX/L=XX/O=generated/CN=generated',
82+
'-keyout', PEM_FILE, '-out', PEM_FILE])
83+
# Restrict access to PEM_FILE
84+
os.chmod(PEM_FILE, stat.S_IRUSR | stat.S_IWUSR)
85+
c.NotebookApp.certfile = PEM_FILE
86+
87+
# Set a password if JUPYTER_PASSWORD is set
88+
if 'JUPYTER_PASSWORD' in os.environ:
89+
c.NotebookApp.password = passwd(os.environ['JUPYTER_PASSWORD'])
90+
del os.environ['JUPYTER_PASSWORD']

0 commit comments

Comments
 (0)