Skip to content

Commit c6e76fe

Browse files
author
Piers Harding
committed
TSK-1823: add example Docker container support
* move requirements.txt to tools dir for Docker * Dockerfile recipe * make commands for notebook, tests, lint
1 parent bccbdd2 commit c6e76fe

5 files changed

Lines changed: 116 additions & 11 deletions

File tree

‎Makefile‎

Lines changed: 50 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -1,11 +1,15 @@
11
# simple makefile to simplify repetitive build env management tasks under posix
22
PYTHON ?= python3
3-
PYLINT ?= pylint3
3+
PYLINT ?= pylint
44
NOSETESTS ?= nosetests3
55
FLAKE ?= flake8
6+
NAME = arl
7+
IMG = $(NAME)_img
8+
DOCKERFILE = Dockerfile
9+
DOCKER=docker
610

711

8-
all: clean inplace test
12+
all: clean nosetests
913

1014
clean:
1115
$(PYTHON) setup.py clean
@@ -15,16 +19,22 @@ in: inplace # just a shortcut
1519
inplace:
1620
$(PYTHON) setup.py install >/dev/null 2>&1 || (echo "'$(PYTHON) setup.py install' failed."; exit -1)
1721

18-
unittest: inplace
22+
build: in
23+
24+
# clean out the cache before tests are run
25+
cleantests:
26+
cd tests && rm -rf __pycache__
27+
28+
unittest: cleantests
1929
$(PYTHON) -m unittest discover -f --locals -s tests -p "test_*.py"
2030

21-
pytest: inplace
31+
pytest: cleantests
2232
pytest -x -v tests/
2333

24-
nosetests: inplace
25-
$(NOSETESTS) -s -v -e create_low_test_beam -e create_low_test_skycomponents_from_gleam tests/
34+
nosetests: cleantests
35+
$(NOSETESTS) -s -v -e create_low_test_beam -e create_low_test_skycomponents_from_gleam tests/
2636

27-
nosetests-coverage: inplace
37+
nosetests-coverage: inplace cleantests
2838
rm -rf coverage .coverage
2939
$(NOSETESTS) -s -v --with-coverage arl
3040

@@ -38,7 +48,7 @@ docs: inplace
3848

3949
code-flake:
4050
# flake8 ignore long lines and trailing whitespace
41-
$(FLAKE) --ignore=E501,W293 arl
51+
$(FLAKE) --ignore=E501,W293,F401 arl
4252

4353
code-lint:
4454
$(PYLINT) --extension-pkg-whitelist=numpy \
@@ -55,3 +65,35 @@ notebook:
5565
IP=`ip a show $${DEVICE} | grep ' inet ' | awk '{print $$2}' | sed 's/\/.*//'` && \
5666
echo "Launching at IP: $${IP}" && \
5767
jupyter notebook --no-browser --ip=$${IP} --port=8888 examples/arl/
68+
69+
docker_build:
70+
cd tools && $(DOCKER) build -t $(IMG) -f $(DOCKERFILE) .
71+
72+
docker_notebook: docker_build
73+
CTNR=`$(DOCKER) ps -q -f name=$(NAME)_notebook` && \
74+
if [ -n "$${CTNR}" ]; then $(DOCKER) rm -f $(NAME)_notebook; fi
75+
DEVICE=`ip link | grep -E " ens| wlan| eth" | grep BROADCAST | tail -1 | cut -d : -f 2 | sed "s/ //"` && \
76+
IP=`ip a show $${DEVICE} | grep ' inet ' | awk '{print $$2}' | sed 's/\/.*//'` && \
77+
echo "Launching at IP: $${IP}" && \
78+
$(DOCKER) run --name $(NAME)_notebook --hostname $(NAME)_notebook --volume $$(pwd):/arl -e IP=$${IP} \
79+
--net=host -p 8888:8888 -p 8787:8787 -p 8788:8788 -p 8789:8789 -d $(IMG)
80+
sleep 3
81+
$(DOCKER) logs $(NAME)_notebook
82+
83+
docker_tests: docker_build cleantests
84+
CTNR=`$(DOCKER) ps -q -f name=$(NAME)_tests` && \
85+
if [ -n "$${CTNR}" ]; then $(DOCKER) rm -f $(NAME)_tests; fi
86+
$(DOCKER) run --rm --name $(NAME)_tests --hostname $(NAME) --volume $$(pwd):/arl \
87+
--net=host -ti $(IMG) /bin/sh -c "cd /arl && make nosetests"
88+
89+
docker_pytest: docker_build cleantests
90+
CTNR=`$(DOCKER) ps -q -f name=$(NAME)_tests` && \
91+
if [ -n "$${CTNR}" ]; then $(DOCKER) rm -f $(NAME)_tests; fi
92+
$(DOCKER) run --rm --name $(NAME)_tests --hostname $(NAME) --volume $$(pwd):/arl \
93+
--net=host -ti $(IMG) /bin/sh -c "cd /arl && make pytest"
94+
95+
docker_lint: docker_build
96+
CTNR=`$(DOCKER) ps -q -f name=$(NAME)_lint` && \
97+
if [ -n "$${CTNR}" ]; then $(DOCKER) rm -f $(NAME)_lint; fi
98+
$(DOCKER) run --rm --name $(NAME)_lint --hostname $(NAME) --volume $$(pwd):/arl \
99+
--net=host -ti $(IMG) /bin/sh -c "cd /arl && make code-analysis"

‎README.md‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -27,7 +27,7 @@ The ARL has a few dependencies:
2727
* Python 3.5+ (Python 2.x may work but it will retire on the 12/04/2020 so best to migrate now - https://pythonclock.org/)
2828
* git 2.11.0+
2929
* git-lfs 2.2.1+
30-
* Python package dependencies as defined in the requirements.txt
30+
* Python package dependencies as defined in the tools/requirements.txt
3131

3232
The Python dependencies will install (amongst other things) Jupyter, numpy, scipy, scikit, and Dask. Because of this it maybe advantageous to setup a virtualenv to contain the project - [instructions can be found here](http://docs.python-guide.org/en/latest/dev/virtualenvs/).
3333

@@ -95,7 +95,7 @@ Common Installation Process
9595
`cd algorithm-reference-library`
9696

9797
* Install required python package::
98-
`pip install -r requirements.txt`
98+
`pip install -r tools/requirements.txt`
9999

100100
* Setup ARL::
101101
`python setup.py install`

‎tools/Dockerfile‎

Lines changed: 59 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,59 @@
1+
# Universal image for running Notebook, Dask pipelines, tests, and lint checkers
2+
FROM ubuntu:16.04
3+
4+
MAINTAINER Piers Harding "piers@catalyst.net.nz"
5+
6+
ENV LANG en_NZ.UTF-8
7+
ENV LANGUAGE en_NZ.UTF-8
8+
ENV LC_ALL en_NZ.UTF-8
9+
ENV HOME /root
10+
ENV DEBIAN_FRONTEND noninteractive
11+
12+
# the package basics for Python 3
13+
RUN \
14+
DEBIAN_FRONTEND=noninteractive apt-get update -y && \
15+
apt-get install -y locales tzdata python-pip net-tools vim-tiny software-properties-common \
16+
python-software-properties build-essential curl wget fonts-liberation ca-certificates && \
17+
echo "Setting locales ..." && /usr/sbin/locale-gen en_US.UTF-8 && \
18+
/usr/sbin/locale-gen en_NZ.UTF-8 && \
19+
echo "Setting timezone ..." && /bin/echo 'Pacific/Auckland' | tee /etc/timezone && DEBIAN_FRONTEND=noninteractive dpkg-reconfigure --frontend noninteractive tzdata && \
20+
add-apt-repository -y ppa:git-core/ppa && \
21+
curl -s https://packagecloud.io/install/repositories/github/git-lfs/script.deb.sh | bash && \
22+
apt-get install -y git-lfs && \
23+
git lfs install && \
24+
apt-get install -y python3-dev python3-pip python3-tk flake8 python3-nose \
25+
virtualenv virtualenvwrapper && \
26+
apt-get install -y graphviz && \
27+
apt-get clean -y
28+
29+
# Install Tini
30+
RUN wget --quiet https://github.com/krallin/tini/releases/download/v0.10.0/tini && \
31+
echo "1361527f39190a7338a0b434bd8c88ff7233ce7b9a4876f3315c22fce7eca1b0 *tini" | sha256sum -c - && \
32+
mv tini /usr/local/bin/tini && \
33+
chmod +x /usr/local/bin/tini
34+
35+
# Add and install Python modules
36+
ADD ./requirements.txt /src/requirements.txt
37+
RUN cd /src; pip3 install --upgrade pip; pip3 install -r requirements.txt
38+
RUN pip3 install bokeh && pip3 install pytest
39+
RUN pip3 install -U pylint
40+
41+
# runtime specific environment
42+
ENV JENKINS_URL 1
43+
ENV PYTHONPATH /arl
44+
ENV JUPYTER_PATH /arl/examples/arl
45+
46+
# Bundle app source
47+
ADD ./boot.sh /
48+
49+
# We share in the arl repo here
50+
VOLUME ["/arl"]
51+
52+
# Expose Jupyter and Bokeh ports
53+
EXPOSE 8888 8787
54+
55+
# Setup the entrypoint or environment
56+
ENTRYPOINT ["tini", "--"]
57+
58+
# Run - default is notebook
59+
CMD ["/boot.sh"]

‎tools/boot.sh‎

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,5 @@
1+
#!/bin/sh
2+
3+
# launch the notebook - the IP address to listen on is passed in via env-var IP
4+
IP=${IP:-0.0.0.0}
5+
jupyter notebook --allow-root --no-browser --ip=${IP} --port=8888 /arl/examples/arl/

requirements.txt renamed to tools/requirements.txt

Lines changed: 0 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -29,4 +29,3 @@ sphinx_rtd_theme
2929
psutil
3030
cloudpickle
3131
seqfile
32-

0 commit comments

Comments
 (0)