Skip to content

Commit 486d13d

Browse files
author
mfarrera
committed
Script adjustments to run on csd3 with slurm
1 parent 31a6d09 commit 486d13d

5 files changed

Lines changed: 189 additions & 6 deletions

File tree

‎scripts/build-arl-csd3.sh‎

Lines changed: 9 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -5,27 +5,31 @@
55
# =================================================== #
66

77
# Load the modules
8-
module load python-3.5.2-gcc-5.4.0-rdp6q5l
9-
#module load python-3.6.1-gcc-5.4.0-23fr5u4
8+
#module load python-3.5.2-gcc-5.4.0-rdp6q5l
9+
module load python-3.6.1-gcc-5.4.0-23fr5u4
1010
module load git-2.14.1-gcc-5.4.0-acb553e
1111
module load git-lfs-2.3.0-gcc-5.4.0-oktvmkw
1212
module load cfitsio-3.410-gcc-5.4.0-tp3pkyv
1313

1414
# Clone the repository
15-
git clone https://github.com/SKA-ScienceDataProcessor/algorithm-reference-library/
16-
cd algorithm-reference-library/
15+
#git clone https://github.com/SKA-ScienceDataProcessor/algorithm-reference-library/
16+
#cd algorithm-reference-library/
1717

1818
# Get the data
1919
git-lfs pull
2020

2121
# Start the building ARL through building a python virtualenvironment
22-
virtualenv -p `which python3` _build
22+
#virtualenv -p `which python3` _build
2323
source _build/bin/activate
2424
pip install --upgrade pip
2525
pip install -U setuptools
2626
pip install coverage numpy
2727
pip install -r requirements.txt
2828

29+
# Install mpi4py
30+
pip install mpi4py
31+
32+
2933
pip install virtualenvwrapper
3034

3135
echo 'Adding the arl and ffiwrappers path to the virtual environment'

‎scripts/csd3-slurm/activate-env-csd3.sh‎

Lines changed: 31 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,31 @@
1+
#!/bin/bash
2+
3+
# =================================================== #
4+
# Loads environment to run ARL from SLURM in CSD3 #
5+
# =================================================== #
6+
7+
# Load the modules
8+
#module load python-3.5.2-gcc-5.4.0-rdp6q5l
9+
module load python-3.6.1-gcc-5.4.0-23fr5u4
10+
#module load git-2.14.1-gcc-5.4.0-acb553e
11+
#module load git-lfs-2.3.0-gcc-5.4.0-oktvmkw
12+
module load cfitsio-3.410-gcc-5.4.0-tp3pkyv
13+
14+
15+
# Start the building ARL through building a python virtualenvironment
16+
#virtualenv -p `which python3` _build
17+
source $ARLROOT/_build/bin/activate
18+
echo source $ARLROOT/_build/bin/activate
19+
20+
# _build/lib/python3.x/site-packages/_virtualenv_path_extensions.pth
21+
#source virtualenvwrapper.sh
22+
#add2virtualenv $PWD
23+
#add2virtualenv $PWD/ffiwrappers/src/
24+
25+
# This is required for some systems (i.e. Jenkins server or macos) others
26+
# detect the python libraries alone and link with correct flags without setting
27+
# up
28+
# the flags explicitely
29+
export LDFLAGS="$(python3-config --ldflags) -lcfitsio"
30+
31+

‎scripts/csd3-slurm/slurm_submit.peta4.skylake‎

Lines changed: 147 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,147 @@
1+
#!/bin/bash
2+
#!
3+
#! Example SLURM job script for Peta4-Skylake (Skylake CPUs, OPA)
4+
#! Last updated: Mon 13 Nov 12:25:17 GMT 2017
5+
#!
6+
7+
#!#############################################################
8+
#!#### Modify the options in this section as appropriate ######
9+
#!#############################################################
10+
11+
#! sbatch directives begin here ###############################
12+
#! Name of the job:
13+
#SBATCH -J cpujob
14+
#! Redirect stderr and stdout
15+
#SBATCH --output results/mpi/mpitest-peta4.%j.out
16+
#SBATCH --error results/mpi/mpitest-peta4.%j.err
17+
#! Which project should be charged (NB Wilkes projects end in '-GPU'):
18+
#SBATCH -A SKA-SDP-SL2-CPU
19+
#! How many whole nodes should be allocated?
20+
#SBATCH --nodes=1
21+
#! How many (MPI) tasks will there be in total? (<=nodes*12)
22+
#! The skylake/skylake-himem nodes have 32 CPUs (cores) each. ntasks=64
23+
#SBATCH --ntasks=12
24+
#! How much wallclock time will be required?
25+
#SBATCH --time=04:22:00
26+
#! What types of email messages do you wish to receive?
27+
#SBATCH --mail-type=FAIL
28+
#! Uncomment this to prevent the job from being requeued (e.g. if
29+
#! interrupted by node failure or system downtime):
30+
##SBATCH --no-requeue
31+
32+
# Request the nodes in exclusive mode
33+
##SBATCH --exclusive
34+
# Request the nodes to be on the same switch
35+
##SBATCH --switches=1
36+
37+
#! For 6GB per CPU, set "-p skylake"; for 12GB per CPU, set "-p skylake-himem":
38+
#SBATCH -p skylake
39+
40+
#! sbatch directives end here (put any additional directives above this line)
41+
42+
#! Notes:
43+
#! Charging is determined by core number*walltime.
44+
#! The --ntasks value refers to the number of tasks to be launched by SLURM only. This
45+
#! usually equates to the number of MPI tasks launched. Reduce this from nodes*32 if
46+
#! demanded by memory requirements, or if OMP_NUM_THREADS>1.
47+
#! Each task is allocated 1 core by default, and each core is allocated 5990MB (skylake)
48+
#! and 12040MB (skylake-himem). If this is insufficient, also specify
49+
#! --cpus-per-task and/or --mem (the latter specifies MB per node).
50+
51+
#! Number of nodes and tasks per node allocated by SLURM (do not change):
52+
numnodes=$SLURM_JOB_NUM_NODES
53+
numtasks=$SLURM_NTASKS
54+
mpi_tasks_per_node=$(echo "$SLURM_TASKS_PER_NODE" | sed -e 's/^\([0-9][0-9]*\).*$/\1/')
55+
#! ############################################################
56+
#! Modify the settings below to specify the application's environment, location
57+
#! and launch method:
58+
59+
#! Optionally modify the environment seen by the application
60+
#! (note that SLURM reproduces the environment at submission irrespective of ~/.bashrc):
61+
. /etc/profile.d/modules.sh # Leave this line (enables the module command)
62+
module purge # Removes all modules still loaded
63+
module load rhel7/default-peta4 # REQUIRED - loads the basic environment
64+
65+
export ARLROOT=/home/mf582/scratch/ARL/algorithm-reference-library
66+
67+
#! Insert additional module load commands after this line if needed:
68+
echo "source ${ARLROOT}/scripts/csd3-slurm/activate-env-csd3.sh"
69+
source $ARLROOT/scripts/csd3-slurm/activate-env-csd3.sh
70+
71+
#module load intel/bundles/psxe/2017.4
72+
73+
#! Full path to application executable:
74+
application="$ARLROOT/workflows/mpi/imaging-pipelines-mpi.py"
75+
76+
#! Run options for the application:
77+
options=""
78+
79+
#! Work directory (i.e. where the job will run):
80+
workdir="$SLURM_SUBMIT_DIR" # The value of SLURM_SUBMIT_DIR sets workdir to the directory
81+
# in which sbatch is run.
82+
83+
#! Are you using OpenMP (NB this is unrelated to OpenMPI)? If so increase this
84+
#! safe value to no more than 32:
85+
export OMP_NUM_THREADS=1
86+
87+
#! Number of MPI tasks to be started by the application per node and in total (do not change):
88+
np=$[${numnodes}*${mpi_tasks_per_node}]
89+
90+
#! The following variables define a sensible pinning strategy for Intel MPI tasks -
91+
#! this should be suitable for both pure MPI and hybrid MPI/OpenMP jobs:
92+
export I_MPI_PIN_DOMAIN=omp:compact # Domains are $OMP_NUM_THREADS cores in size
93+
export I_MPI_PIN_ORDER=scatter # Adjacent domains have minimal sharing of caches/sockets
94+
#! Notes:
95+
#! 1. These variables influence Intel MPI only.
96+
#! 2. Domains are non-overlapping sets of cores which map 1-1 to MPI tasks.
97+
#! 3. I_MPI_PIN_PROCESSOR_LIST is ignored if I_MPI_PIN_DOMAIN is set.
98+
#! 4. If MPI tasks perform better when sharing caches/sockets, try I_MPI_PIN_ORDER=compact.
99+
100+
101+
#! Uncomment one choice for CMD below (add mpirun/mpiexec options if necessary):
102+
103+
#! Choose this for a MPI code (possibly using OpenMP) using Intel MPI.
104+
#CMD="mpirun -ppn $mpi_tasks_per_node -np $np $application $options"
105+
#CMD="mpiexec -trace -np $np python $application $options"
106+
CMD="mpiexec -np $np python $application $options"
107+
#CMD="mpirun -ppn $mpi_tasks_per_node -np $np $application $options"
108+
#mpi_tasks_per_node=$[${mpi_tasks_per_node}/2]
109+
#CMD2="mpirun -ppn $mpi_tasks_per_node -np $np $application $options"
110+
111+
#! Choose this for a pure shared-memory OpenMP parallel program on a single node:
112+
#! (OMP_NUM_THREADS threads will be created):
113+
#CMD="$application $options"
114+
115+
#! Choose this for a MPI code (possibly using OpenMP) using OpenMPI:
116+
#CMD="mpirun -npernode $mpi_tasks_per_node -np $np $application $options"
117+
118+
119+
###############################################################
120+
### You should not have to change anything below this line ####
121+
###############################################################
122+
123+
cd $workdir
124+
echo -e "Changed directory to `pwd`.\n"
125+
126+
JOBID=$SLURM_JOB_ID
127+
128+
echo -e "JobID: $JOBID\n======"
129+
echo "Time: `date`"
130+
echo "Running on master node: `hostname`"
131+
echo "Current directory: `pwd`"
132+
133+
if [ "$SLURM_JOB_NODELIST" ]; then
134+
#! Create a machine file:
135+
export NODEFILE=`generate_pbs_nodefile`
136+
cat $NODEFILE | uniq > machine.file.$JOBID
137+
echo -e "\nNodes allocated:\n================"
138+
echo `cat machine.file.$JOBID | sed -e 's/\..*$//g'`
139+
fi
140+
141+
echo -e "\nnumtasks=$numtasks, numnodes=$numnodes, mpi_tasks_per_node=$mpi_tasks_per_node (OMP_NUM_THREADS=$OMP_NUM_THREADS)"
142+
143+
echo -e "\nExecuting command:\n==================\n$CMD\n"
144+
145+
eval $CMD
146+
#echo -e "\nExecuting command:\n==================\n$CMD2\n"
147+
#eval $CMD2

‎workflows/mpi/results/mpi/foo‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1 @@
1+
This folder needs to exist for the scripts to run, as they put their output here.

‎workflows/mpi/run-mpi.sh‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,2 +1,2 @@
11
#!/usr/bin/env bash
2-
mpiexec -n $1 python $2
2+
mpiexec -n $1 -f machinefile python $2

0 commit comments

Comments
 (0)