Repository navigation
Expand file tree
/
Copy pathfunctions-common
More file actions
2512 lines (2207 loc) · 74.9 KB
/
Copy pathfunctions-common
File metadata and controls
2512 lines (2207 loc) · 74.9 KB
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
499
500
501
502
503
504
505
506
507
508
509
510
511
512
513
514
515
516
517
518
519
520
521
522
523
524
525
526
527
528
529
530
531
532
533
534
535
536
537
538
539
540
541
542
543
544
545
546
547
548
549
550
551
552
553
554
555
556
557
558
559
560
561
562
563
564
565
566
567
568
569
570
571
572
573
574
575
576
577
578
579
580
581
582
583
584
585
586
587
588
589
590
591
592
593
594
595
596
597
598
599
600
601
602
603
604
605
606
607
608
609
610
611
612
613
614
615
616
617
618
619
620
621
622
623
624
625
626
627
628
629
630
631
632
633
634
635
636
637
638
639
640
641
642
643
644
645
646
647
648
649
650
651
652
653
654
655
656
657
658
659
660
661
662
663
664
665
666
667
668
669
670
671
672
673
674
675
676
677
678
679
680
681
682
683
684
685
686
687
688
689
690
691
692
693
694
695
696
697
698
699
700
701
702
703
704
705
706
707
708
709
710
711
712
713
714
715
716
717
718
719
720
721
722
723
724
725
726
727
728
729
730
731
732
733
734
735
736
737
738
739
740
741
742
743
744
745
746
747
748
749
750
751
752
753
754
755
756
757
758
759
760
761
762
763
764
765
766
767
768
769
770
771
772
773
774
775
776
777
778
779
780
781
782
783
784
785
786
787
788
789
790
791
792
793
794
795
796
797
798
799
800
801
802
803
804
805
806
807
808
809
810
811
812
813
814
815
816
817
818
819
820
821
822
823
824
825
826
827
828
829
830
831
832
833
834
835
836
837
838
839
840
841
842
843
844
845
846
847
848
849
850
851
852
853
854
855
856
857
858
859
860
861
862
863
864
865
866
867
868
869
870
871
872
873
874
875
876
877
878
879
880
881
882
883
884
885
886
887
888
889
890
891
892
893
894
895
896
897
898
899
900
901
902
903
904
905
906
907
908
909
910
911
912
913
914
915
916
917
918
919
920
921
922
923
924
925
926
927
928
929
930
931
932
933
934
935
936
937
938
939
940
941
942
943
944
945
946
947
948
949
950
951
952
953
954
955
956
957
958
959
960
961
962
963
964
965
966
967
968
969
970
971
972
973
974
975
976
977
978
979
980
981
982
983
984
985
986
987
988
989
990
991
992
993
994
995
996
997
998
999
1000
#!/bin/bash
#
# functions-common - Common functions used by DevStack components
#
# The canonical copy of this file is maintained in the DevStack repo.
# All modifications should be made there and then sync'ed to other repos
# as required.
#
# This file is sorted alphabetically within the function groups.
#
# - Config Functions
# - Control Functions
# - Distro Functions
# - Git Functions
# - OpenStack Functions
# - Package Functions
# - Process Functions
# - Service Functions
# - System Functions
#
# The following variables are assumed to be defined by certain functions:
#
# - ``ENABLED_SERVICES``
# - ``ERROR_ON_CLONE``
# - ``FILES``
# - ``OFFLINE``
# - ``RECLONE``
# - ``REQUIREMENTS_DIR``
# - ``STACK_USER``
# - ``http_proxy``, ``https_proxy``, ``no_proxy``
#
# Save trace setting
_XTRACE_FUNCTIONS_COMMON=$(set +o | grep xtrace)
set +o xtrace
# ensure we don't re-source this in the same environment
[[ -z "$_DEVSTACK_FUNCTIONS_COMMON" ]] || return 0
declare -r -g _DEVSTACK_FUNCTIONS_COMMON=1
# Global Config Variables
declare -A -g GITREPO
declare -A -g GITBRANCH
declare -A -g GITDIR
KILL_PATH="$(which kill)"
# Save these variables to .stackenv
STACK_ENV_VARS="BASE_SQL_CONN DATA_DIR DEST ENABLED_SERVICES HOST_IP \
KEYSTONE_SERVICE_URI \
LOGFILE OS_CACERT SERVICE_HOST STACK_USER TLS_IP \
HOST_IPV6 SERVICE_IP_VERSION TUNNEL_ENDPOINT_IP TUNNEL_IP_VERSION"
# Saves significant environment variables to .stackenv for later use
# Refers to a lot of globals, only TOP_DIR and STACK_ENV_VARS are required to
# function, the rest are simply saved and do not cause problems if they are undefined.
# save_stackenv [tag]
function save_stackenv {
local tag=${1:-""}
# Save some values we generated for later use
time_stamp=$(date "+$TIMESTAMP_FORMAT")
echo "# $time_stamp $tag" >$TOP_DIR/.stackenv
for i in $STACK_ENV_VARS; do
echo $i=${!i} >>$TOP_DIR/.stackenv
done
}
# Update/create user clouds.yaml file.
# clouds.yaml will have
# - A `devstack` entry for the `demo` user for the `demo` project.
# - A `devstack-admin` entry for the `admin` user for the `admin` project.
# write_clouds_yaml
function write_clouds_yaml {
# The location is a variable to allow for easier refactoring later to make it
# overridable. There is currently no usecase where doing so makes sense, so
# it's not currently configurable.
CLOUDS_YAML=/etc/openstack/clouds.yaml
sudo mkdir -p $(dirname $CLOUDS_YAML)
sudo chown -R $STACK_USER /etc/openstack
CA_CERT_ARG=''
if [ -f "$SSL_BUNDLE_FILE" ]; then
CA_CERT_ARG="--os-cacert $SSL_BUNDLE_FILE"
fi
# devstack: user with the member role on demo project
$PYTHON $TOP_DIR/tools/update_clouds_yaml.py \
--file $CLOUDS_YAML \
--os-cloud devstack \
--os-region-name $REGION_NAME \
$CA_CERT_ARG \
--os-auth-url $KEYSTONE_SERVICE_URI \
--os-username demo \
--os-password $ADMIN_PASSWORD \
--os-project-name demo
# devstack-admin: user with the admin role on the admin project
$PYTHON $TOP_DIR/tools/update_clouds_yaml.py \
--file $CLOUDS_YAML \
--os-cloud devstack-admin \
--os-region-name $REGION_NAME \
$CA_CERT_ARG \
--os-auth-url $KEYSTONE_SERVICE_URI \
--os-username admin \
--os-password $ADMIN_PASSWORD \
--os-project-name admin
# devstack-admin-demo: user with the admin role on the demo project
$PYTHON $TOP_DIR/tools/update_clouds_yaml.py \
--file $CLOUDS_YAML \
--os-cloud devstack-admin-demo \
--os-region-name $REGION_NAME \
$CA_CERT_ARG \
--os-auth-url $KEYSTONE_SERVICE_URI \
--os-username admin \
--os-password $ADMIN_PASSWORD \
--os-project-name demo
# devstack-alt: user with the member role on alt_demo project
$PYTHON $TOP_DIR/tools/update_clouds_yaml.py \
--file $CLOUDS_YAML \
--os-cloud devstack-alt \
--os-region-name $REGION_NAME \
$CA_CERT_ARG \
--os-auth-url $KEYSTONE_SERVICE_URI \
--os-username alt_demo \
--os-password $ADMIN_PASSWORD \
--os-project-name alt_demo
# devstack-alt-member: user with the member role on alt_demo project
$PYTHON $TOP_DIR/tools/update_clouds_yaml.py \
--file $CLOUDS_YAML \
--os-cloud devstack-alt-member \
--os-region-name $REGION_NAME \
$CA_CERT_ARG \
--os-auth-url $KEYSTONE_SERVICE_URI \
--os-username alt_demo_member \
--os-password $ADMIN_PASSWORD \
--os-project-name alt_demo
# devstack-alt-reader: user with the reader role on alt_demo project
$PYTHON $TOP_DIR/tools/update_clouds_yaml.py \
--file $CLOUDS_YAML \
--os-cloud devstack-alt-reader \
--os-region-name $REGION_NAME \
$CA_CERT_ARG \
--os-auth-url $KEYSTONE_SERVICE_URI \
--os-username alt_demo_reader \
--os-password $ADMIN_PASSWORD \
--os-project-name alt_demo
# devstack-reader: user with the reader role on demo project
$PYTHON $TOP_DIR/tools/update_clouds_yaml.py \
--file $CLOUDS_YAML \
--os-cloud devstack-reader \
--os-region-name $REGION_NAME \
$CA_CERT_ARG \
--os-auth-url $KEYSTONE_SERVICE_URI \
--os-username demo_reader \
--os-password $ADMIN_PASSWORD \
--os-project-name demo
# devstack-system-admin: user with the admin role on the system
$PYTHON $TOP_DIR/tools/update_clouds_yaml.py \
--file $CLOUDS_YAML \
--os-cloud devstack-system-admin \
--os-region-name $REGION_NAME \
$CA_CERT_ARG \
--os-auth-url $KEYSTONE_SERVICE_URI \
--os-username admin \
--os-password $ADMIN_PASSWORD \
--os-system-scope all
# devstack-system-member: user with the member role on the system
$PYTHON $TOP_DIR/tools/update_clouds_yaml.py \
--file $CLOUDS_YAML \
--os-cloud devstack-system-member \
--os-region-name $REGION_NAME \
$CA_CERT_ARG \
--os-auth-url $KEYSTONE_SERVICE_URI \
--os-username system_member \
--os-password $ADMIN_PASSWORD \
--os-system-scope all
# devstack-system-reader: user with the reader role on the system
$PYTHON $TOP_DIR/tools/update_clouds_yaml.py \
--file $CLOUDS_YAML \
--os-cloud devstack-system-reader \
--os-region-name $REGION_NAME \
$CA_CERT_ARG \
--os-auth-url $KEYSTONE_SERVICE_URI \
--os-username system_reader \
--os-password $ADMIN_PASSWORD \
--os-system-scope all
cat >> $CLOUDS_YAML <<EOF
functional:
image_name: $DEFAULT_IMAGE_NAME
EOF
# CLean up any old clouds.yaml files we had laying around
rm -f $(eval echo ~"$STACK_USER")/.config/openstack/clouds.yaml
}
# trueorfalse <True|False> <VAR>
#
# Normalize config-value provided in variable VAR to either "True" or
# "False". If VAR is unset (i.e. $VAR evaluates as empty), the value
# of the second argument will be used as the default value.
#
# Accepts as False: 0 no No NO false False FALSE
# Accepts as True: 1 yes Yes YES true True TRUE
#
# usage:
# VAL=$(trueorfalse False VAL)
function trueorfalse {
local xtrace
xtrace=$(set +o | grep xtrace)
set +o xtrace
local default=$1
if [ -z $2 ]; then
die $LINENO "variable to normalize required"
fi
local testval=${!2:-}
case "$testval" in
"1" | [yY]es | "YES" | [tT]rue | "TRUE" ) echo "True" ;;
"0" | [nN]o | "NO" | [fF]alse | "FALSE" ) echo "False" ;;
* ) echo "$default" ;;
esac
$xtrace
}
function isset {
[[ -v "$1" ]]
}
# Control Functions
# =================
# Prints backtrace info
# filename:lineno:function
# backtrace level
function backtrace {
local level=$1
local deep
deep=$((${#BASH_SOURCE[@]} - 1))
echo "[Call Trace]"
while [ $level -le $deep ]; do
echo "${BASH_SOURCE[$deep]}:${BASH_LINENO[$deep-1]}:${FUNCNAME[$deep-1]}"
deep=$((deep - 1))
done
}
# Prints line number and "message" then exits
# die $LINENO "message"
function die {
local exitcode=$?
set +o xtrace
local line=$1; shift
if [ $exitcode == 0 ]; then
exitcode=1
fi
backtrace 2
err $line "$*"
# Give buffers a second to flush
sleep 1
exit $exitcode
}
# Checks an environment variable is not set or has length 0 OR if the
# exit code is non-zero and prints "message" and exits
# NOTE: env-var is the variable name without a '$'
# die_if_not_set $LINENO env-var "message"
function die_if_not_set {
local exitcode=$?
local xtrace
xtrace=$(set +o | grep xtrace)
set +o xtrace
local line=$1; shift
local evar=$1; shift
if ! is_set $evar || [ $exitcode != 0 ]; then
die $line "$*"
fi
$xtrace
}
function deprecated {
local text=$1
DEPRECATED_TEXT+="\n$text"
echo "WARNING: $text" >&2
}
# Prints line number and "message" in error format
# err $LINENO "message"
function err {
local exitcode=$?
local xtrace
xtrace=$(set +o | grep xtrace)
set +o xtrace
local msg="[ERROR] ${BASH_SOURCE[2]}:$1 $2"
echo "$msg" 1>&2;
if [[ -n ${LOGDIR} ]]; then
echo "$msg" >> "${LOGDIR}/error.log"
fi
$xtrace
return $exitcode
}
# Checks an environment variable is not set or has length 0 OR if the
# exit code is non-zero and prints "message"
# NOTE: env-var is the variable name without a '$'
# err_if_not_set $LINENO env-var "message"
function err_if_not_set {
local exitcode=$?
local xtrace
xtrace=$(set +o | grep xtrace)
set +o xtrace
local line=$1; shift
local evar=$1; shift
if ! is_set $evar || [ $exitcode != 0 ]; then
err $line "$*"
fi
$xtrace
return $exitcode
}
# Exit after outputting a message about the distribution not being supported.
# exit_distro_not_supported [optional-string-telling-what-is-missing]
function exit_distro_not_supported {
if [[ -z "$DISTRO" ]]; then
GetDistro
fi
if [ $# -gt 0 ]; then
die $LINENO "Support for $DISTRO is incomplete: no support for $@"
else
die $LINENO "Support for $DISTRO is incomplete."
fi
}
# Test if the named environment variable is set and not zero length
# is_set env-var
function is_set {
local var=\$"$1"
eval "[ -n \"$var\" ]" # For ex.: sh -c "[ -n \"$var\" ]" would be better, but several exercises depends on this
}
# Prints line number and "message" in warning format
# warn $LINENO "message"
function warn {
local exitcode=$?
local xtrace
xtrace=$(set +o | grep xtrace)
set +o xtrace
local msg="[WARNING] ${BASH_SOURCE[2]}:$1 $2"
echo "$msg"
$xtrace
return $exitcode
}
# Distro Functions
# ================
# Determine OS Vendor, Release and Update
#
# NOTE : For portability, you almost certainly do not want to use
# these variables directly! The "is_*" functions defined below this
# bundle up compatible platforms under larger umbrellas that we have
# determinted are compatible enough (e.g. is_ubuntu covers Ubuntu &
# Debian, is_fedora covers RPM-based distros). Higher-level functions
# such as "install_package" further abstract things in better ways.
#
# ``os_VENDOR`` - vendor name: ``Ubuntu``, ``Fedora``, etc
# ``os_RELEASE`` - major release: ``16.04`` (Ubuntu), ``23`` (Fedora)
# ``os_PACKAGE`` - package type: ``deb`` or ``rpm``
# ``os_CODENAME`` - vendor's codename for release: ``xenial``
declare -g os_VENDOR os_RELEASE os_PACKAGE os_CODENAME
# Make a *best effort* attempt to install lsb_release packages for the
# user if not available. Note can't use generic install_package*
# because they depend on this!
function _ensure_lsb_release {
if [[ -x $(command -v lsb_release 2>/dev/null) ]]; then
return
fi
if [[ -x $(command -v apt-get 2>/dev/null) ]]; then
sudo apt-get install -y lsb-release
elif [[ -x $(command -v zypper 2>/dev/null) ]]; then
sudo zypper -n install lsb-release
elif [[ -x $(command -v dnf 2>/dev/null) ]]; then
sudo dnf install -y redhat-lsb-core || sudo dnf install -y openeuler-lsb
else
die $LINENO "Unable to find or auto-install lsb_release"
fi
}
# GetOSVersion
# Set the following variables:
# - os_RELEASE
# - os_CODENAME
# - os_VENDOR
# - os_PACKAGE
function GetOSVersion {
# CentOS Stream 9 does not provide lsb_release
source /etc/os-release
if [[ "${ID}${VERSION}" == "centos9" ]]; then
os_RELEASE=${VERSION_ID}
os_CODENAME="n/a"
os_VENDOR=$(echo $NAME | tr -d '[:space:]')
elif [[ "${ID}${VERSION}" =~ "rocky9" ]]; then
os_VENDOR="Rocky"
os_RELEASE=${VERSION_ID}
else
_ensure_lsb_release
os_RELEASE=$(lsb_release -r -s)
os_CODENAME=$(lsb_release -c -s)
os_VENDOR=$(lsb_release -i -s)
fi
if [[ $os_VENDOR =~ (Debian|Ubuntu) ]]; then
os_PACKAGE="deb"
else
os_PACKAGE="rpm"
fi
typeset -xr os_VENDOR
typeset -xr os_RELEASE
typeset -xr os_PACKAGE
typeset -xr os_CODENAME
}
# Translate the OS version values into common nomenclature
# Sets global ``DISTRO`` from the ``os_*`` values
declare -g DISTRO
function GetDistro {
GetOSVersion
if [[ "$os_VENDOR" =~ (Ubuntu) || "$os_VENDOR" =~ (Debian) ]]; then
# 'Everyone' refers to Ubuntu / Debian releases by
# the code name adjective
DISTRO=$os_CODENAME
elif [[ "$os_VENDOR" =~ (Fedora) ]]; then
# For Fedora, just use 'f' and the release
DISTRO="f$os_RELEASE"
elif [[ "$os_VENDOR" =~ (Red.*Hat) || \
"$os_VENDOR" =~ (CentOS) || \
"$os_VENDOR" =~ (AlmaLinux) || \
"$os_VENDOR" =~ (Scientific) || \
"$os_VENDOR" =~ (OracleServer) || \
"$os_VENDOR" =~ (Rocky) || \
"$os_VENDOR" =~ (Virtuozzo) ]]; then
# Drop the . release as we assume it's compatible
# XXX re-evaluate when we get RHEL10
DISTRO="rhel${os_RELEASE::1}"
elif [[ "$os_VENDOR" =~ (openEuler) ]]; then
DISTRO="openEuler-$os_RELEASE"
else
# We can't make a good choice here. Setting a sensible DISTRO
# is part of the problem, but not the major issue -- we really
# only use DISTRO in the code as a fine-filter.
#
# The bigger problem is categorising the system into one of
# our two big categories as Ubuntu/Debian-ish or
# Fedora/CentOS-ish.
#
# The setting of os_PACKAGE above is only set to "deb" based
# on a hard-coded list of vendor names ... thus we will
# default to thinking unknown distros are RPM based
# (ie. is_ubuntu does not match). But the platform will then
# also not match in is_fedora, because that also has a list of
# names.
#
# So, if you are reading this, getting your distro supported
# is really about making sure it matches correctly in these
# functions. Then you can choose a sensible way to construct
# DISTRO based on your distros release approach.
die $LINENO "Unable to determine DISTRO, can not continue."
fi
typeset -xr DISTRO
}
# Utility function for checking machine architecture
# is_arch arch-type
function is_arch {
[[ "$(uname -m)" == "$1" ]]
}
# Determine if current distribution is an Oracle distribution
# is_oraclelinux
function is_oraclelinux {
if [[ -z "$os_VENDOR" ]]; then
GetOSVersion
fi
[ "$os_VENDOR" = "OracleServer" ]
}
# Determine if current distribution is a Fedora-based distribution
# (Fedora, RHEL, CentOS, Rocky, etc).
# is_fedora
function is_fedora {
if [[ -z "$os_VENDOR" ]]; then
GetOSVersion
fi
[ "$os_VENDOR" = "Fedora" ] || [ "$os_VENDOR" = "Red Hat" ] || \
[ "$os_VENDOR" = "openEuler" ] || \
[ "$os_VENDOR" = "RedHatEnterpriseServer" ] || \
[ "$os_VENDOR" = "RedHatEnterprise" ] || \
[ "$os_VENDOR" = "Rocky" ] || \
[ "$os_VENDOR" = "CentOS" ] || [ "$os_VENDOR" = "CentOSStream" ] || \
[ "$os_VENDOR" = "AlmaLinux" ] || \
[ "$os_VENDOR" = "OracleServer" ] || [ "$os_VENDOR" = "Virtuozzo" ]
}
# Determine if current distribution is an Ubuntu-based distribution
# It will also detect non-Ubuntu but Debian-based distros
# is_ubuntu
function is_ubuntu {
if [[ -z "$os_PACKAGE" ]]; then
GetOSVersion
fi
[ "$os_PACKAGE" = "deb" ]
}
# Determine if current distribution is an openEuler distribution
# is_openeuler
function is_openeuler {
if [[ -z "$os_PACKAGE" ]]; then
GetOSVersion
fi
[ "$os_VENDOR" = "openEuler" ]
}
# Git Functions
# =============
# Returns openstack release name for a given branch name
# ``get_release_name_from_branch branch-name``
function get_release_name_from_branch {
local branch=$1
if [[ $branch =~ "stable/" || $branch =~ "proposed/" ]]; then
echo ${branch#*/}
else
echo "master"
fi
}
# git clone only if directory doesn't exist already. Since ``DEST`` might not
# be owned by the installation user, we create the directory and change the
# ownership to the proper user.
# Set global ``RECLONE=yes`` to simulate a clone when dest-dir exists
# Set global ``ERROR_ON_CLONE=True`` to abort execution with an error if the git repo
# does not exist (default is False, meaning the repo will be cloned).
# Uses globals ``ERROR_ON_CLONE``, ``OFFLINE``, ``RECLONE``
# git_clone remote dest-dir branch
function git_clone {
local git_remote=$1
local git_dest=$2
local git_ref=$3
local orig_dir
orig_dir=$(pwd)
local git_clone_flags=""
RECLONE=$(trueorfalse False RECLONE)
if [[ "${GIT_DEPTH}" -gt 0 ]]; then
git_clone_flags="$git_clone_flags --depth $GIT_DEPTH"
fi
if [[ "$OFFLINE" = "True" ]]; then
echo "Running in offline mode, clones already exist"
# print out the results so we know what change was used in the logs
cd $git_dest
git show --oneline | head -1
cd $orig_dir
return
fi
if echo $git_ref | egrep -q "^refs"; then
# If our branch name is a gerrit style refs/changes/...
if [[ ! -d $git_dest ]]; then
if [[ "$ERROR_ON_CLONE" = "True" ]]; then
echo "The $git_dest project was not found; if this is a gate job, add"
echo "the project to 'required-projects' in the job definition."
die $LINENO "ERROR_ON_CLONE is set to True so cloning not allowed in this configuration"
fi
git_timed clone $git_clone_flags $git_remote $git_dest
fi
cd $git_dest
git_timed fetch $git_remote $git_ref && git checkout FETCH_HEAD
else
# do a full clone only if the directory doesn't exist
if [[ ! -d $git_dest ]]; then
if [[ "$ERROR_ON_CLONE" = "True" ]]; then
echo "The $git_dest project was not found; if this is a gate job, add"
echo "the project to the \$PROJECTS variable in the job definition."
die $LINENO "ERROR_ON_CLONE is set to True so cloning not allowed in this configuration"
fi
git_timed clone $git_clone_flags $git_remote $git_dest
cd $git_dest
git checkout $git_ref
elif [[ "$RECLONE" = "True" ]]; then
# if it does exist then simulate what clone does if asked to RECLONE
cd $git_dest
# set the url to pull from and fetch
git remote set-url origin $git_remote
git_timed fetch origin
# remove the existing ignored files (like pyc) as they cause breakage
# (due to the py files having older timestamps than our pyc, so python
# thinks the pyc files are correct using them)
sudo find $git_dest -name '*.pyc' -delete
# handle git_ref accordingly to type (tag, branch)
if [[ -n "`git show-ref refs/tags/$git_ref`" ]]; then
git_update_tag $git_ref
elif [[ -n "`git show-ref refs/heads/$git_ref`" ]]; then
git_update_branch $git_ref
elif [[ -n "`git show-ref refs/remotes/origin/$git_ref`" ]]; then
git_update_remote_branch $git_ref
else
die $LINENO "$git_ref is neither branch nor tag"
fi
fi
fi
# NOTE(ianw) 2022-04-13 : commit [1] has broken many assumptions
# about how we clone and work with repos. Mark them safe globally
# as a work-around.
#
# NOTE(danms): On bionic (and likely others) git-config may write
# ~stackuser/.gitconfig if not run with sudo -H. Using --system
# writes these changes to /etc/gitconfig which is more
# discoverable anyway.
#
# [1] https://github.com/git/git/commit/8959555cee7ec045958f9b6dd62e541affb7e7d9
sudo git config --system --add safe.directory ${git_dest}
# print out the results so we know what change was used in the logs
cd $git_dest
git show --oneline | head -1
cd $orig_dir
}
# A variation on git clone that lets us specify a project by it's
# actual name, like oslo.config. This is exceptionally useful in the
# library installation case
function git_clone_by_name {
local name=$1
local repo=${GITREPO[$name]}
local dir=${GITDIR[$name]}
local branch=${GITBRANCH[$name]}
git_clone $repo $dir $branch
}
# git can sometimes get itself infinitely stuck with transient network
# errors or other issues with the remote end. This wraps git in a
# timeout/retry loop and is intended to watch over non-local git
# processes that might hang. GIT_TIMEOUT, if set, is passed directly
# to timeout(1); otherwise the default value of 0 maintains the status
# quo of waiting forever.
# usage: git_timed <git-command>
function git_timed {
local count=0
local timeout=0
if [[ -n "${GIT_TIMEOUT}" ]]; then
timeout=${GIT_TIMEOUT}
fi
time_start "git_timed"
until timeout -s SIGINT ${timeout} git "$@"; do
# 124 is timeout(1)'s special return code when it reached the
# timeout; otherwise assume fatal failure
if [[ $? -ne 124 ]]; then
die $LINENO "git call failed: [git $@]"
fi
count=$(($count + 1))
warn $LINENO "timeout ${count} for git call: [git $@]"
if [ $count -eq 3 ]; then
die $LINENO "Maximum of 3 git retries reached"
fi
sleep 5
done
time_stop "git_timed"
}
# git update using reference as a branch.
# git_update_branch ref
function git_update_branch {
local git_branch=$1
git checkout -f origin/$git_branch
# a local branch might not exist
git branch -D $git_branch || true
git checkout -b $git_branch
}
# git update using reference as a branch.
# git_update_remote_branch ref
function git_update_remote_branch {
local git_branch=$1
git checkout -b $git_branch -t origin/$git_branch
}
# git update using reference as a tag. Be careful editing source at that repo
# as working copy will be in a detached mode
# git_update_tag ref
function git_update_tag {
local git_tag=$1
git tag -d $git_tag
# fetching given tag only
git_timed fetch origin tag $git_tag
git checkout -f $git_tag
}
# OpenStack Functions
# ===================
# Get the default value for HOST_IP
# get_default_host_ip fixed_range floating_range host_ip_iface host_ip
function get_default_host_ip {
local fixed_range=$1
local floating_range=$2
local host_ip_iface=$3
local host_ip=$4
local af=$5
# Search for an IP unless an explicit is set by ``HOST_IP`` environment variable
if [ -z "$host_ip" -o "$host_ip" == "dhcp" ]; then
host_ip=""
# Find the interface used for the default route
host_ip_iface=${host_ip_iface:-$(ip -f $af route | awk '/default/ {print $5}' | head -1)}
local host_ips
host_ips=$(LC_ALL=C ip -f $af addr show ${host_ip_iface} | sed /temporary/d |awk /$af'/ {split($2,parts,"/"); print parts[1]}')
local ip
for ip in $host_ips; do
# Attempt to filter out IP addresses that are part of the fixed and
# floating range. Note that this method only works if the ``netaddr``
# python library is installed. If it is not installed, an error
# will be printed and the first IP from the interface will be used.
# If that is not correct set ``HOST_IP`` in ``localrc`` to the correct
# address.
if [[ "$af" == "inet6" ]]; then
host_ip=$ip
break;
fi
if ! (address_in_net $ip $fixed_range || address_in_net $ip $floating_range); then
host_ip=$ip
break;
fi
done
fi
echo $host_ip
}
# Generates hex string from ``size`` byte of pseudo random data
# generate_hex_string size
function generate_hex_string {
local size=$1
hexdump -n "$size" -v -e '/1 "%02x"' /dev/urandom
}
# Grab a numbered field from python prettytable output
# Fields are numbered starting with 1
# Reverse syntax is supported: -1 is the last field, -2 is second to last, etc.
# get_field field-number
function get_field {
local data field
while read data; do
if [ "$1" -lt 0 ]; then
field="(\$(NF$1))"
else
field="\$$(($1 + 1))"
fi
echo "$data" | awk -F'[ \t]*\\|[ \t]*' "{print $field}"
done
}
# install default policy
# copy over a default policy.json and policy.d for projects
function install_default_policy {
local project=$1
local project_uc
project_uc=$(echo $1|tr a-z A-Z)
local conf_dir="${project_uc}_CONF_DIR"
# eval conf dir to get the variable
conf_dir="${!conf_dir}"
local project_dir="${project_uc}_DIR"
# eval project dir to get the variable
project_dir="${!project_dir}"
local sample_conf_dir="${project_dir}/etc/${project}"
local sample_policy_dir="${project_dir}/etc/${project}/policy.d"
# first copy any policy.json
cp -p $sample_conf_dir/policy.json $conf_dir
# then optionally copy over policy.d
if [[ -d $sample_policy_dir ]]; then
cp -r $sample_policy_dir $conf_dir/policy.d
fi
}
# Add a policy to a policy.json file
# Do nothing if the policy already exists
# ``policy_add policy_file policy_name policy_permissions``
function policy_add {
local policy_file=$1
local policy_name=$2
local policy_perm=$3
if grep -q ${policy_name} ${policy_file}; then
echo "Policy ${policy_name} already exists in ${policy_file}"
return
fi
# Add a terminating comma to policy lines without one
# Remove the closing '}' and all lines following to the end-of-file
local tmpfile
tmpfile=$(mktemp)
uniq ${policy_file} | sed -e '
s/]$/],/
/^[}]/,$d
' > ${tmpfile}
# Append policy and closing brace
echo " \"${policy_name}\": ${policy_perm}" >>${tmpfile}
echo "}" >>${tmpfile}
mv ${tmpfile} ${policy_file}
}
# Gets or creates a domain
# Usage: get_or_create_domain <name> <description>
function get_or_create_domain {
local domain_id
domain_id=$(
openstack --os-cloud devstack-system-admin domain create $1 \
--description "$2" --or-show \
-f value -c id
)
echo $domain_id
}
# Gets or creates group
# Usage: get_or_create_group <groupname> <domain> [<description>]
function get_or_create_group {
local desc="${3:-}"
local group_id
# Gets group id
group_id=$(
# Creates new group with --or-show
openstack --os-cloud devstack-system-admin group create $1 \
--domain $2 --description "$desc" --or-show \
-f value -c id
)
echo $group_id
}
# Gets or creates user
# Usage: get_or_create_user <username> <password> <domain> [<email>]
function get_or_create_user {
local user_id
if [[ ! -z "$4" ]]; then
local email="--email=$4"
else
local email=""
fi
# Gets user id
user_id=$(
# Creates new user with --or-show
openstack --os-cloud devstack-system-admin user create \
$1 \
--password "$2" \
--domain=$3 \
$email \
--or-show \
-f value -c id
)
echo $user_id
}
# Gets or creates project
# Usage: get_or_create_project <name> <domain>
function get_or_create_project {
local project_id
project_id=$(
# Creates new project with --or-show
openstack --os-cloud devstack-system-admin project create $1 \
--domain=$2 \
--or-show -f value -c id
)
echo $project_id
}
# Gets or creates role
# Usage: get_or_create_role <name>
function get_or_create_role {
local role_id
role_id=$(
# Creates role with --or-show
openstack --os-cloud devstack-system-admin role create $1 \
--or-show -f value -c id
)
echo $role_id
}
# Returns the domain parts of a function call if present
# Usage: _get_domain_args [<user_domain> <project_domain>]
function _get_domain_args {
local domain
domain=""
if [[ -n "$1" ]]; then
domain="$domain --user-domain $1"
fi
if [[ -n "$2" ]]; then
domain="$domain --project-domain $2"
fi
echo $domain
}
# Gets or adds user role to project
# Usage: get_or_add_user_project_role <role> <user> <project> [<user_domain> <project_domain>]
function get_or_add_user_project_role {
local user_role_id
local domain_args
domain_args=$(_get_domain_args $4 $5)
# Note this is idempotent so we are safe across multiple
# duplicate calls.
openstack --os-cloud devstack-system-admin role add $1 \
--user $2 \
--project $3 \
$domain_args
user_role_id=$(openstack --os-cloud devstack-system-admin role assignment list \
--role $1 \
--user $2 \
--project $3 \
$domain_args \
-c Role -f value)
echo $user_role_id
}
# Gets or adds user role to domain
# Usage: get_or_add_user_domain_role <role> <user> <domain>
function get_or_add_user_domain_role {
local user_role_id
# Note this is idempotent so we are safe across multiple
# duplicate calls.
openstack --os-cloud devstack-system-admin role add $1 \
--user $2 \
--domain $3
user_role_id=$(openstack --os-cloud devstack-system-admin role assignment list \
--role $1 \
--user $2 \
--domain $3 \
-c Role -f value)
echo $user_role_id
}
# Gets or adds user role to system
# Usage: get_or_add_user_system_role <role> <user> <system> [<user_domain>]
function get_or_add_user_system_role {
local user_role_id
local domain_args
domain_args=$(_get_domain_args $4)
# Gets user role id
user_role_id=$(openstack --os-cloud devstack-system-admin role assignment list \
--role $1 \
--user $2 \
--system $3 \
$domain_args \
-f value -c Role)
if [[ -z "$user_role_id" ]]; then
# Adds role to user and get it
openstack --os-cloud devstack-system-admin role add $1 \