Skip to content

Commit bc673e4

Browse files
author
mfarrera
committed
Other scripts to run scaling tests
1 parent 486d13d commit bc673e4

4 files changed

Lines changed: 287 additions & 0 deletions

File tree

‎scripts/csd3-slurm/create-slurm-jobs.sh‎

Lines changed: 47 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,47 @@
1+
#! /bin/bash
2+
# skylake nodes have 6GB per CPU and 12 cores per node
3+
# skylake-himem have 12GB per CPU and 32 CPUs per node
4+
# Script should be run from $ARLROOT/scripts/csd3-slurm
5+
6+
TEMPLATE=slurm_submit.peta4-skylake.template
7+
OUTPUT=slurm_job.peta4
8+
NUMNODES_LIST='2 4 8 16'
9+
NODETYPE_LIST='skylake-himem'
10+
NFREQ_LIST='203 407'
11+
12+
JOB_FOLDER=../../scripts/csd3-slurm/
13+
WORK_DIRECTORY=../../workflows/mpi
14+
15+
for NUMNODES in $NUMNODES_LIST
16+
do
17+
for NFREQ in $NFREQ_LIST
18+
do
19+
for NODETYPE in $NODETYPE_LIST
20+
do
21+
# echo $NUMNODES $NFREQ $NODETYPE
22+
if [ $NODETYPE == 'skylake' ]
23+
then
24+
NUMCORES_LIST='6 12'
25+
elif [ $NODETYPE == 'skylake-himem' ]
26+
then
27+
NUMCORES_LIST='16 32'
28+
else
29+
echo 'Wrong nodetype'
30+
fi
31+
for NUMCORES in $NUMCORES_LIST
32+
do
33+
NUMTASKS=$((NUMNODES*NUMCORES))
34+
sed -e "s/@@NUMNODES@@/$NUMNODES/" -e "s/@@NUMTASKS@@/$NUMTASKS/" -e "s/@@NFREQ@@/$NFREQ/" -e "s/@@NODETYPE@@/$NODETYPE/" $TEMPLATE > $OUTPUT.$NODETYPE.$NUMNODES.$NUMTASKS.$NFREQ
35+
echo 'Created' $OUTPUT.$NODETYPE.$NUMNODES.$NUMTASKS.$NFREQ
36+
cd $WORK_DIRECTORY
37+
echo 'Current directory' $PWD
38+
echo 'Submiting' $JOB_FOLDER$OUTPUT.$NODETYPE.$NUMNODES.$NUMTASKS.$NFREQ
39+
sbatch $JOB_FOLDER$OUTPUT.$NODETYPE.$NUMNODES.$NUMTASKS.$NFREQ
40+
# rm $OUTPUT.$NODETYPE.$NUMNODES.$NUMTASKS.$NFREQ
41+
cd $JOB_FOLDER
42+
done
43+
done
44+
done
45+
done
46+
47+

‎scripts/csd3-slurm/slurm_submit.peta4-skylake.template‎

Lines changed: 148 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,148 @@
1+
#!/bin/bash
2+
#!
3+
#! Example SLURM job script for Peta4-Skylake (Skylake CPUs, OPA)
4+
#! Last updated: Mon 13 Nov 12:25:17 GMT 2017
5+
#!
6+
7+
#!#############################################################
8+
#!#### Modify the options in this section as appropriate ######
9+
#!#############################################################
10+
11+
#! sbatch directives begin here ###############################
12+
#! Name of the job:
13+
#SBATCH -J cpujob
14+
#! Redirect stderr and stdout
15+
#SBATCH --output results/mpi/mpitest-@@NODETYPE@@-@@NUMNODES@@-@@NUMTASKS@@-@@NFREQ@@.out
16+
#SBATCH --error results/mpi/mpitest-@@NODETYPE@@-@@NUMNODES@@-@@NUMTASKS@@-@@NFREQ@@.err
17+
#! Which project should be charged (NB Wilkes projects end in '-GPU'):
18+
#SBATCH -A SKA-SDP-SL2-CPU
19+
#! How many whole nodes should be allocated?
20+
#SBATCH --nodes=@@NUMNODES@@
21+
#! How many (MPI) tasks will there be in total? (<=nodes*12)
22+
#! The skylake/skylake-himem nodes have 32 CPUs (cores) each. ntasks=64
23+
#SBATCH --ntasks=@@NUMTASKS@@
24+
#! How much wallclock time will be required?
25+
#SBATCH --time=04:22:00
26+
#! What types of email messages do you wish to receive?
27+
#SBATCH --mail-type=FAIL
28+
#! Uncomment this to prevent the job from being requeued (e.g. if
29+
#! interrupted by node failure or system downtime):
30+
##SBATCH --no-requeue
31+
32+
# Request the nodes in exclusive mode
33+
#SBATCH --exclusive
34+
# Request the nodes to be on the same switch
35+
#SBATCH --switches=1
36+
37+
#! For 6GB per CPU, set "-p skylake"; for 12GB per CPU, set "-p skylake-himem":
38+
#SBATCH -p @@NODETYPE@@
39+
40+
#! sbatch directives end here (put any additional directives above this line)
41+
42+
#! Notes:
43+
#! Charging is determined by core number*walltime.
44+
#! The --ntasks value refers to the number of tasks to be launched by SLURM only. This
45+
#! usually equates to the number of MPI tasks launched. Reduce this from nodes*32 if
46+
#! demanded by memory requirements, or if OMP_NUM_THREADS>1.
47+
#! Each task is allocated 1 core by default, and each core is allocated 5990MB (skylake)
48+
#! and 12040MB (skylake-himem). If this is insufficient, also specify
49+
#! --cpus-per-task and/or --mem (the latter specifies MB per node).
50+
51+
#! Number of nodes and tasks per node allocated by SLURM (do not change):
52+
numnodes=$SLURM_JOB_NUM_NODES
53+
numtasks=$SLURM_NTASKS
54+
mpi_tasks_per_node=$(echo "$SLURM_TASKS_PER_NODE" | sed -e 's/^\([0-9][0-9]*\).*$/\1/')
55+
#! ############################################################
56+
#! Modify the settings below to specify the application's environment, location
57+
#! and launch method:
58+
59+
#! Optionally modify the environment seen by the application
60+
#! (note that SLURM reproduces the environment at submission irrespective of ~/.bashrc):
61+
. /etc/profile.d/modules.sh # Leave this line (enables the module command)
62+
module purge # Removes all modules still loaded
63+
module load rhel7/default-peta4 # REQUIRED - loads the basic environment
64+
65+
export ARLROOT=/home/mf582/scratch/ARL/algorithm-reference-library
66+
67+
#! Insert additional module load commands after this line if needed:
68+
echo "source ${ARLROOT}/scripts/csd3-slurm/activate-env-csd3.sh"
69+
source $ARLROOT/scripts/csd3-slurm/activate-env-csd3.sh
70+
71+
#module load intel/bundles/psxe/2017.4
72+
73+
#! Full path to application executable:
74+
application="$ARLROOT/workflows/mpi/imaging-pipelines-mpi.py"
75+
76+
#! Run options for the application:
77+
options="--nfreqwin=@@NFREQ@@"
78+
79+
#! Work directory (i.e. where the job will run):
80+
workdir="$SLURM_SUBMIT_DIR" # The value of SLURM_SUBMIT_DIR sets workdir to the directory
81+
# in which sbatch is run.
82+
#workdir="$ARLROOT/workflows/mpi"
83+
84+
#! Are you using OpenMP (NB this is unrelated to OpenMPI)? If so increase this
85+
#! safe value to no more than 32:
86+
export OMP_NUM_THREADS=1
87+
88+
#! Number of MPI tasks to be started by the application per node and in total (do not change):
89+
np=$[${numnodes}*${mpi_tasks_per_node}]
90+
91+
#! The following variables define a sensible pinning strategy for Intel MPI tasks -
92+
#! this should be suitable for both pure MPI and hybrid MPI/OpenMP jobs:
93+
export I_MPI_PIN_DOMAIN=omp:compact # Domains are $OMP_NUM_THREADS cores in size
94+
export I_MPI_PIN_ORDER=scatter # Adjacent domains have minimal sharing of caches/sockets
95+
#! Notes:
96+
#! 1. These variables influence Intel MPI only.
97+
#! 2. Domains are non-overlapping sets of cores which map 1-1 to MPI tasks.
98+
#! 3. I_MPI_PIN_PROCESSOR_LIST is ignored if I_MPI_PIN_DOMAIN is set.
99+
#! 4. If MPI tasks perform better when sharing caches/sockets, try I_MPI_PIN_ORDER=compact.
100+
101+
102+
#! Uncomment one choice for CMD below (add mpirun/mpiexec options if necessary):
103+
104+
#! Choose this for a MPI code (possibly using OpenMP) using Intel MPI.
105+
#CMD="mpirun -ppn $mpi_tasks_per_node -np $np $application $options"
106+
#CMD="mpiexec -trace -np $np python $application $options"
107+
CMD="mpiexec -np $np python $application $options"
108+
#CMD="mpirun -ppn $mpi_tasks_per_node -np $np $application $options"
109+
#mpi_tasks_per_node=$[${mpi_tasks_per_node}/2]
110+
#CMD2="mpirun -ppn $mpi_tasks_per_node -np $np $application $options"
111+
112+
#! Choose this for a pure shared-memory OpenMP parallel program on a single node:
113+
#! (OMP_NUM_THREADS threads will be created):
114+
#CMD="$application $options"
115+
116+
#! Choose this for a MPI code (possibly using OpenMP) using OpenMPI:
117+
#CMD="mpirun -npernode $mpi_tasks_per_node -np $np $application $options"
118+
119+
120+
###############################################################
121+
### You should not have to change anything below this line ####
122+
###############################################################
123+
124+
cd $workdir
125+
echo -e "Changed directory to `pwd`.\n"
126+
127+
JOBID=$SLURM_JOB_ID
128+
129+
echo -e "JobID: $JOBID\n======"
130+
echo "Time: `date`"
131+
echo "Running on master node: `hostname`"
132+
echo "Current directory: `pwd`"
133+
134+
if [ "$SLURM_JOB_NODELIST" ]; then
135+
#! Create a machine file:
136+
export NODEFILE=`generate_pbs_nodefile`
137+
cat $NODEFILE | uniq > machine.file.$JOBID
138+
echo -e "\nNodes allocated:\n================"
139+
echo `cat machine.file.$JOBID | sed -e 's/\..*$//g'`
140+
fi
141+
142+
echo -e "\nnumtasks=$numtasks, numnodes=$numnodes, mpi_tasks_per_node=$mpi_tasks_per_node (OMP_NUM_THREADS=$OMP_NUM_THREADS)"
143+
144+
echo -e "\nExecuting command:\n==================\n$CMD\n"
145+
146+
eval $CMD
147+
#echo -e "\nExecuting command:\n==================\n$CMD2\n"
148+
#eval $CMD2

‎workflows/mpi/process-slurm-job-results.sh‎

Lines changed: 50 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,50 @@
1+
#! /bin/bash
2+
# skylake nodes have 6GB per CPU and 12 cores per node
3+
# skylake-himem have 12GB per CPU and 32 CPUs per node
4+
# Script should be run from $ARLROOT/scripts/csd3-slurm
5+
6+
#TEMPLATE=slurm_submit.peta4-skylake.template
7+
#OUTPUT=slurm_job.peta4
8+
NUMNODES_LIST='2 4'
9+
NODETYPE_LIST='skylake-himem'
10+
NFREQ_LIST='71 101'
11+
12+
RESULTS_OUTPUT=mpitest
13+
14+
#JOB_FOLDER=../../scripts/csd3-slurm/
15+
#WORK_DIRECTORY=../../workflows/mpi
16+
17+
cd results/mpi
18+
echo 'Current directory' $PWD
19+
for NUMNODES in $NUMNODES_LIST
20+
do
21+
for NFREQ in $NFREQ_LIST
22+
do
23+
for NODETYPE in $NODETYPE_LIST
24+
do
25+
# echo $NUMNODES $NFREQ $NODETYPE
26+
if [ $NODETYPE == 'skylake' ]
27+
then
28+
NUMCORES_LIST='6 12'
29+
elif [ $NODETYPE == 'skylake-himem' ]
30+
then
31+
NUMCORES_LIST='16 32'
32+
else
33+
echo 'Wrong nodetype'
34+
fi
35+
for NUMCORES in $NUMCORES_LIST
36+
do
37+
NUMTASKS=$((NUMNODES*NUMCORES))
38+
#sed -e "s/@@NUMNODES@@/$NUMNODES/" -e "s/@@NUMTASKS@@/$NUMTASKS/" -e "s/@@NFREQ@@/$NFREQ/" -e "s/@@NODETYPE@@/$NODETYPE/" $TEMPLATE > $OUTPUT.$NODETYPE.$NUMNODES.$NUMTASKS.$NFREQ
39+
#cat $RESULTS_OUTPUT-$NODETYPE-$NUMNODES-$NUMTASKS-$NFREQ.out |grep "0: predict finished in" | awk {'print $NUMNODES $NUMTASKS $NFREQ $5'}
40+
cat $RESULTS_OUTPUT-$NODETYPE-$NUMNODES-$NUMTASKS-$NFREQ.out |grep ^"0: predict finished in" | awk {"print ${NUMNODES} \"\t\" ${NUMTASKS} \"\t\" $NFREQ \"\t\" \$5"} >> predict-results.txt
41+
cat $RESULTS_OUTPUT-$NODETYPE-$NUMNODES-$NUMTASKS-$NFREQ.out |grep ^"0: invert finished in" | awk {"print ${NUMNODES} \"\t\" ${NUMTASKS} \"\t\" $NFREQ \"\t\" \$5"} >> invert-results.txt
42+
cat $RESULTS_OUTPUT-$NODETYPE-$NUMNODES-$NUMTASKS-$NFREQ.out |grep ^"0: continuum imaging finished in" | awk {"print ${NUMNODES} \"\t\" ${NUMTASKS} \"\t\" $NFREQ \"\t\" \$6"} >> contimg-results.txt
43+
cat $RESULTS_OUTPUT-$NODETYPE-$NUMNODES-$NUMTASKS-$NFREQ.out |grep ^"0: ical finished in" | awk {"print ${NUMNODES} \"\t\" ${NUMTASKS} \"\t\" $NFREQ \"\t\" \$5"} >> ical-results.txt
44+
45+
done
46+
done
47+
done
48+
done
49+
50+

‎workflows/mpi/submit-slurm-jobs.sh‎

Lines changed: 42 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,42 @@
1+
#! /bin/bash
2+
# skylake nodes have 6GB per CPU and 12 cores per node
3+
# skylake-himem have 12GB per CPU and 32 CPUs per node
4+
TEMPLATE=slurm_submit.peta4-skylake.template
5+
OUTPUT=slurm_job.peta4
6+
NUMNODES_LIST='2 4 8 16'
7+
NODETYPE_LIST='skylake skylake-himem'
8+
NFREQ_LIST='21 41 71 101 203'
9+
10+
JOB_FOLDER=../../scripts/csd3-slurm/
11+
WORK_DIRECTORY=../../workflows/mpi
12+
13+
for NUMNODES in $NUMNODES_LIST
14+
do
15+
for NFREQ in $NFREQ_LIST
16+
do
17+
for NODETYPE in $NODETYPE_LIST
18+
do
19+
# echo $NUMNODES $NFREQ $NODETYPE
20+
if [ $NODETYPE == 'skylake' ]
21+
then
22+
NUMCORES_LIST='6 12'
23+
elif [ $NODETYPE == 'skylake-himem' ]
24+
then
25+
NUMCORES_LIST='16 32'
26+
else
27+
echo 'Wrong nodetype'
28+
fi
29+
30+
for NUMCORES in $NUMCORES_LIST
31+
do
32+
NUMTASKS=$((NUMNODES*NUMCORES))
33+
#sed -e "s/@@NUMNODES@@/$NUMNODES/" -e "s/@@NUMTASKS@@/$NUMTASKS/" -e "s/@@NFREQ@@/$NFREQ/" -e "s/@@NODETYPE@@/$NODETYPE/" $TEMPLATE > $OUTPUT.$NODETYPE.$NUMNODES.$NUMTASKS.$NFREQ
34+
echo 'Submiting' $OUTPUT.$NODETYPE.$NUMNODES.$NUMTASKS.$NFREQ
35+
# sbatch $OUTPUT.$NODETYPE.$NUMNODES.$NUMTASKS.$NFREQ
36+
# rm $OUTPUT.$NODETYPE.$NUMNODES.$NUMTASKS.$NFREQ
37+
done
38+
done
39+
done
40+
done
41+
42+

0 commit comments

Comments
 (0)