Skip to content

feat: build option --no-aux to disable downloading of all auxiliary data #1374

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
c-dilks merged 5 commits into from
Aug 10, 2026
Merged

feat: build option --no-aux to disable downloading of all auxiliary data #1374

Changes from 1 commit
Commits
Show all changes
5 commits
Select commit Hold shift + click to select a range
e71c65c
feat: build option `--no-aux` to disable downloading of all auxiliary…
c-dilks Aug 4, 2026
f64c3c7
ci: respect rename
c-dilks Aug 4, 2026
f6d0442
fix: macOS and ancient bash...
c-dilks Aug 4, 2026
253b8fe
fix: macOS bash lacks associative arrays
c-dilks Aug 4, 2026
19d2efd
fix: arg printout
c-dilks Aug 10, 2026
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Next Next commit
feat: build option --no-aux to disable downloading of all auxiliary… 
… data

I also renamed `--nomaps` -> `--no-maps`, _etc._; the old names still work as legacy aliases,
so no changes are needed downstream.
  • Loading branch information
@c-dilks
c-dilks committed Aug 10, 2026
commit e71c65cccbba772a455bac988de01054a64f04f8
113 changes: 68 additions & 45 deletions build-coatjava.sh
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -11,13 +11,15 @@ set -o pipefail

cleanBuild=false
runSpotBugs=false
downloadMaps=true
downloadNets=true
downloadSqlites=true
runUnitTests=false
dataRetrieval=lfs
installClara=false
downloadData=false
auxRetrieval=lfs
declare -A auxDownload=(
[maps]=true
[nets]=true
[sqlites]=true
)

################################################################################
# usage
Expand All @@ -32,17 +34,18 @@ GENERAL OPTIONS
--no-progress no download progress printouts
--help show this message

DATA RETRIEVAL OPTIONS
AUXILIARY DATA RETRIEVAL OPTIONS
How to retrieve magnetic field maps, neural network models, etc.
Choose only one; default is `--'$dataRetrieval'`
Choose only one; default is `--'$auxRetrieval'`
--lfs use Git Large File Storage (requires `git-lfs`)
--cvmfs use CernVM-FS (requires `/cvfms`)
--https use clasweb HTTPS (field maps only)
Additional options
--nomaps do not download/overwrite field maps
--nonets do not download/overwrite neural networks
--nosqlites do not download/overwrite SQLite files for CCDB/RCDB
--wipe remove retrieved data
--wipe remove ALL retrieved auxiliary data and exit
Options for disabling downloads:
--no-aux ignore ALL auxiliary data
--no-maps ignore field maps
--no-nets ignore neural networks
--no-sqlites ignore CCDB/RCDB SQLite files

TESTING OPTIONS
--spotbugs also run spotbugs plugin
Expand All @@ -64,17 +67,8 @@ wgetArgs=()
for xx in $@
do
case $xx in
--spotbugs) runSpotBugs=true ;;
-n) runSpotBugs=false ;;
--nomaps) downloadMaps=false ;;
--nonets) downloadNets=false ;;
--nosqlites) downloadSqlites=false ;;
--unittests) runUnitTests=true ;;
--clean) cleanBuild=true ;;
--depana)
echo "ERROR: option \`$xx\` has been removed; dependency tree printout and analysis now happen automatically in the Maven build lifecycle" >&2
exit 1
;;
--clara) installClara=true ;;
--clean) cleanBuild=true ;;
--quiet)
mvnArgs+=(--quiet --batch-mode)
wgetArgs+=(--quiet)
Expand All @@ -83,12 +77,26 @@ do
mvnArgs+=(--no-transfer-progress)
wgetArgs+=(--no-verbose)
;;
--cvmfs) dataRetrieval=cvmfs ;;
--lfs) dataRetrieval=lfs ;;
--https) dataRetrieval=https ;;
--wipe) dataRetrieval=wipe ;;
--clara) installClara=true ;;
--data) downloadData=true ;;
--lfs) auxRetrieval=lfs ;;
--cvmfs) auxRetrieval=cvmfs ;;
--https) auxRetrieval=https ;;
--wipe) auxRetrieval=wipe ;;
--no-aux|--noaux)
for key in "${!auxDownload[@]}"; do
auxDownload[$key]=false
done
;;
--no-maps|--nomaps) auxDownload['maps']=false ;;
--no-nets|--nonets) auxDownload['nets']=false ;;
--no-sqlites|--nosqlites) auxDownload['sqlites']=false ;;
--spotbugs) runSpotBugs=true ;;
-n) runSpotBugs=false ;;
--unittests) runUnitTests=true ;;
--data) downloadData=true ;;
--depana)
echo "ERROR: option \`$xx\` has been removed; dependency tree printout and analysis now happen automatically in the Maven build lifecycle" >&2
exit 1
;;
--xrootd)
echo "ERROR: option \`$xx\` has been removed; use \`--help\` for guidance" >&2
exit 1
Expand All @@ -101,6 +109,21 @@ do
esac
done

echo "---------------------- ARGUMENTS ----------------------"
print_arg() { printf " %30s = %s\n" "$1" "$2"; }
print_arg "cleanBuild" "$cleanBuild"
print_arg "runSpotBugs" "$runSpotBugs"
print_arg "runUnitTests" "$runUnitTests"
print_arg "installClara" "$installClara"
print_arg "downloadData" "$downloadData"
print_arg "auxRetrieval" "$auxRetrieval"
for key in "${!auxDownload[@]}"; do
print_arg "auxDownload[$key]" "${auxDownload[$key]}"
done
print_arg "mvnArgs" "${mvnArgs[@]:-}"
print_arg "wgetArgs" "${wgetArgs[@]:-}"
echo "-------------------------------------------------------"


################################################################################
# setup
Expand Down Expand Up @@ -142,13 +165,13 @@ if $cleanBuild; then
fi

# wipe retrieved data (field maps, NN models, etc.)
if [ "$dataRetrieval" = "wipe" ]; then
if [ "$auxRetrieval" = "wipe" ]; then
git submodule deinit --all --force
fi

# print cleanup note and exit
if $cleanBuild || [ "$dataRetrieval" = "wipe" ]; then
[ "$dataRetrieval" = "wipe" ] && echo "[+] REMOVED RETRIEVED DATA" || echo "[+] NOTE: retrieved data not removed; use \`--wipe\` if you need to remove them"
if $cleanBuild || [ "$auxRetrieval" = "wipe" ]; then
[ "$auxRetrieval" = "wipe" ] && echo "[+] REMOVED RETRIEVED DATA" || echo "[+] NOTE: retrieved data not removed; use \`--wipe\` if you need to remove them"
$cleanBuild && echo "[+] DONE CLEANING; rerun without \`--clean\` to build"
exit
fi
Expand All @@ -164,7 +187,7 @@ command_exists () {
}

# check data-retrieval options, and prepare accordingly
case $dataRetrieval in
case $auxRetrieval in
lfs)
if ! command_exists git-lfs ; then
echo 'ERROR: `git-lfs` not found; please install it, or use a different data-retrieval option other than `--lfs`' >&2
Expand All @@ -182,7 +205,7 @@ case $dataRetrieval in
https)
;;
*)
echo "ERROR: data retrieval option '$dataRetrieval' is not supported" >&2
echo "ERROR: data retrieval option \`--$auxRetrieval\` is not supported" >&2
exit 1
;;
esac
Expand All @@ -202,7 +225,7 @@ download_lfs() {
# download a magnetic field map
download_map () {
ret=0
case $dataRetrieval in
case $auxRetrieval in
cvmfs)
notify_retrieval 'field map' 'cvmfs'
cp $1 ./
Expand All @@ -226,23 +249,23 @@ download_map () {
;;
*)
ret=1
echo "ERROR::::::::::: called 'download_map' with bad 'dataRetrieval'." >&2
echo "ERROR::::::::::: called 'download_map' with bad 'auxRetrieval' option \`--$auxRetrieval\`." >&2
;;
esac
return $ret
}

# download the default field maps, as defined in libexec/env.sh:
# (and duplicated in etc/services/reconstruction.yaml):
if $downloadMaps; then
case $dataRetrieval in
if ${auxDownload['maps']}; then
case $auxRetrieval in
lfs)
notify_retrieval 'field maps' 'lfs'
download_lfs etc/data/magfield
;;
cvmfs|https)
webDir=https://clasweb.jlab.org/clas12offline/magfield
if [ "$dataRetrieval" = "cvmfs" ]; then
if [ "$auxRetrieval" = "cvmfs" ]; then
webDir=/cvmfs/oasis.opensciencegrid.org/jlab/hallb/clas12/sw/noarch/data/magfield
fi
mkdir -p $magfield_dir
Expand All @@ -253,22 +276,22 @@ if $downloadMaps; then
if [ $? -ne 0 ]; then
echo "ERROR::::::::::: Could not download field map:" >&2
echo "$webDir/$map" >&2
echo "One option is to download manually into etc/data/magfield and then run this build script with --nomaps" >&2
echo "One option is to download manually into \`etc/data/magfield\` and then run this build script with \`--no-maps\`" >&2
exit 1
fi
done
cd -
;;
*)
echo "ERROR: data retrieval option '$dataRetrieval' not supported for field maps" >&2
echo "ERROR: data retrieval option \`--$auxRetrieval\` not supported for field maps" >&2
exit 1
;;
esac
fi

# download neural networks
if $downloadNets; then
case $dataRetrieval in
if ${auxDownload['nets']}; then
case $auxRetrieval in
lfs)
notify_retrieval 'neural networks' 'lfs'
download_lfs etc/data/nnet
Expand All @@ -285,8 +308,8 @@ if $downloadNets; then
fi

# download neural networks
if $downloadSqlites; then
case $dataRetrieval in
if ${auxDownload['sqlites']}; then
case $auxRetrieval in
lfs)
notify_retrieval 'ccdb/rcdb SQLite files' 'lfs'
download_lfs etc/data/sqlite
Expand All @@ -299,7 +322,7 @@ fi

# download validation data
if $downloadData; then
case $dataRetrieval in
case $auxRetrieval in
lfs)
notify_retrieval 'validation data' 'lfs'
download_lfs validation/advanced-tests/data
Expand Down