# lib/cinder

# Install and start **Cinder** volume service

# Dependencies:

#

# - functions

# - DEST, DATA_DIR, STACK_USER must be defined

# - SERVICE_{TENANT_NAME|PASSWORD} must be defined

# - ``KEYSTONE_TOKEN_FORMAT`` must be defined

# stack.sh

# ---------

# - install_cinder

# - configure_cinder

# - init_cinder

# - start_cinder

# - stop_cinder

# - cleanup_cinder

# Save trace setting

XTRACE=$(set +o | grep xtrace)

set +o xtrace

# Defaults

# --------

# set up default driver

CINDER_DRIVER=${CINDER_DRIVER:-default}

CINDER_PLUGINS=$TOP_DIR/lib/cinder_plugins

CINDER_BACKENDS=$TOP_DIR/lib/cinder_backends

# grab plugin config if specified via cinder_driver

if [[ -r $CINDER_PLUGINS/$CINDER_DRIVER ]]; then

source $CINDER_PLUGINS/$CINDER_DRIVER

fi

# set up default directories

CINDER_DIR=$DEST/cinder

CINDERCLIENT_DIR=$DEST/python-cinderclient

CINDER_STATE_PATH=${CINDER_STATE_PATH:=$DATA_DIR/cinder}

CINDER_AUTH_CACHE_DIR=${CINDER_AUTH_CACHE_DIR:-/var/cache/cinder}

CINDER_CONF_DIR=/etc/cinder

CINDER_CONF=$CINDER_CONF_DIR/cinder.conf

CINDER_API_PASTE_INI=$CINDER_CONF_DIR/api-paste.ini

# Public facing bits

if is_ssl_enabled_service "cinder" || is_service_enabled tls-proxy; then

CINDER_SERVICE_PROTOCOL="https"

fi

CINDER_SERVICE_HOST=${CINDER_SERVICE_HOST:-$SERVICE_HOST}

CINDER_SERVICE_PORT=${CINDER_SERVICE_PORT:-8776}

CINDER_SERVICE_PORT_INT=${CINDER_SERVICE_PORT_INT:-18776}

CINDER_SERVICE_PROTOCOL=${CINDER_SERVICE_PROTOCOL:-$SERVICE_PROTOCOL}

# Support entry points installation of console scripts

if [[ -d $CINDER_DIR/bin ]]; then

CINDER_BIN_DIR=$CINDER_DIR/bin

else

CINDER_BIN_DIR=$(get_python_exec_prefix)

fi

# Maintain this here for backward-compatibility with the old configuration

# DEPRECATED: Use CINDER_ENABLED_BACKENDS instead

# Support for multi lvm backend configuration (default is no support)

CINDER_MULTI_LVM_BACKEND=$(trueorfalse False $CINDER_MULTI_LVM_BACKEND)

# Default backends

# The backend format is type:name where type is one of the supported backend

# types (lvm, nfs, etc) and name is the identifier used in the Cinder

# configuration and for the volume type name. Multiple backends are

# comma-separated.

if [[ $CINDER_MULTI_LVM_BACKEND == "False" ]]; then

CINDER_ENABLED_BACKENDS=${CINDER_ENABLED_BACKENDS:-lvm:lvmdriver-1}

else

CINDER_ENABLED_BACKENDS=${CINDER_ENABLED_BACKENDS:-lvm:lvmdriver-1,lvm:lvmdriver-2}

fi

# Should cinder perform secure deletion of volumes?

# Defaults to true, can be set to False to avoid this bug when testing:

# https://bugs.launchpad.net/ubuntu/+source/linux/+bug/1023755

CINDER_SECURE_DELETE=`trueorfalse True $CINDER_SECURE_DELETE`

# Cinder reports allocations back to the scheduler on periodic intervals

# it turns out we can get an "out of space" issue when we run tests too

# quickly just because cinder didn't realize we'd freed up resources.

# Make this configurable so that devstack-gate/tempest can set it to

# less than the 60 second default

# https://bugs.launchpad.net/cinder/+bug/1180976

CINDER_PERIODIC_INTERVAL=${CINDER_PERIODIC_INTERVAL:-60}

# Tell Tempest this project is present

TEMPEST_SERVICES+=,cinder

# Source the enabled backends

if is_service_enabled c-vol && [[ -n "$CINDER_ENABLED_BACKENDS" ]]; then

for be in ${CINDER_ENABLED_BACKENDS//,/ }; do

be_type=${be%%:*}

be_name=${be##*:}

if [[ -r $CINDER_BACKENDS/${be_type} ]]; then

source $CINDER_BACKENDS/${be_type}

fi

done

fi

# Functions

# ---------

# Test if any Cinder services are enabled

# is_cinder_enabled

function is_cinder_enabled {

[[ ,${ENABLED_SERVICES} =~ ,"c-" ]] && return 0

return 1

}

# cleanup_cinder() - Remove residual data files, anything left over from previous

# runs that a clean run would need to clean up

function cleanup_cinder {

# ensure the volume group is cleared up because fails might

# leave dead volumes in the group

local targets=$(sudo tgtadm --op show --mode target)

if [ $? -ne 0 ]; then

# If tgt driver isn't running this won't work obviously

# So check the response and restart if need be

echo "tgtd seems to be in a bad state, restarting..."

if is_ubuntu; then

restart_service tgt

else

restart_service tgtd

fi

targets=$(sudo tgtadm --op show --mode target)

fi

if [[ -n "$targets" ]]; then

local iqn_list=( $(grep --no-filename -r iqn $SCSI_PERSIST_DIR | sed 's/<target //' | sed 's/>//') )

for i in "${iqn_list[@]}"; do

echo removing iSCSI target: $i

sudo tgt-admin --delete $i

done

fi

if is_ubuntu; then

stop_service tgt

else

stop_service tgtd

fi

if is_service_enabled c-vol && [[ -n "$CINDER_ENABLED_BACKENDS" ]]; then

local be be_name be_type

for be in ${CINDER_ENABLED_BACKENDS//,/ }; do

be_type=${be%%:*}

be_name=${be##*:}

if type cleanup_cinder_backend_${be_type} >/dev/null 2>&1; then

cleanup_cinder_backend_${be_type} ${be_name}

fi

done

fi

}

# configure_cinder_rootwrap() - configure Cinder's rootwrap

function configure_cinder_rootwrap {

# Set the paths of certain binaries

local cinder_rootwrap=$(get_rootwrap_location cinder)

# Deploy new rootwrap filters files (owned by root).

# Wipe any existing rootwrap.d files first

if [[ -d $CINDER_CONF_DIR/rootwrap.d ]]; then

sudo rm -rf $CINDER_CONF_DIR/rootwrap.d

fi

# Deploy filters to /etc/cinder/rootwrap.d

sudo mkdir -m 755 $CINDER_CONF_DIR/rootwrap.d

sudo cp $CINDER_DIR/etc/cinder/rootwrap.d/*.filters $CINDER_CONF_DIR/rootwrap.d

sudo chown -R root:root $CINDER_CONF_DIR/rootwrap.d

sudo chmod 644 $CINDER_CONF_DIR/rootwrap.d/*

# Set up rootwrap.conf, pointing to /etc/cinder/rootwrap.d

sudo cp $CINDER_DIR/etc/cinder/rootwrap.conf $CINDER_CONF_DIR/

sudo sed -e "s:^filters_path=.*$:filters_path=$CINDER_CONF_DIR/rootwrap.d:" -i $CINDER_CONF_DIR/rootwrap.conf

sudo chown root:root $CINDER_CONF_DIR/rootwrap.conf

sudo chmod 0644 $CINDER_CONF_DIR/rootwrap.conf

# Specify rootwrap.conf as first parameter to rootwrap

ROOTWRAP_CSUDOER_CMD="$cinder_rootwrap $CINDER_CONF_DIR/rootwrap.conf *"

# Set up the rootwrap sudoers for cinder

local tempfile=`mktemp`

echo "$STACK_USER ALL=(root) NOPASSWD: $ROOTWRAP_CSUDOER_CMD" >$tempfile

chmod 0440 $tempfile

sudo chown root:root $tempfile

sudo mv $tempfile /etc/sudoers.d/cinder-rootwrap

}

# configure_cinder() - Set config files, create data dirs, etc

function configure_cinder {

if [[ ! -d $CINDER_CONF_DIR ]]; then

sudo mkdir -p $CINDER_CONF_DIR

fi

sudo chown $STACK_USER $CINDER_CONF_DIR

cp -p $CINDER_DIR/etc/cinder/policy.json $CINDER_CONF_DIR

configure_cinder_rootwrap

cp $CINDER_DIR/etc/cinder/api-paste.ini $CINDER_API_PASTE_INI

inicomment $CINDER_API_PASTE_INI filter:authtoken auth_host

inicomment $CINDER_API_PASTE_INI filter:authtoken auth_port

inicomment $CINDER_API_PASTE_INI filter:authtoken auth_protocol

inicomment $CINDER_API_PASTE_INI filter:authtoken cafile

inicomment $CINDER_API_PASTE_INI filter:authtoken admin_tenant_name

inicomment $CINDER_API_PASTE_INI filter:authtoken admin_user

inicomment $CINDER_API_PASTE_INI filter:authtoken admin_password

inicomment $CINDER_API_PASTE_INI filter:authtoken signing_dir

configure_auth_token_middleware $CINDER_CONF cinder $CINDER_AUTH_CACHE_DIR

iniset $CINDER_CONF DEFAULT auth_strategy keystone

iniset $CINDER_CONF DEFAULT debug $ENABLE_DEBUG_LOG_LEVEL

iniset $CINDER_CONF DEFAULT verbose True

iniset $CINDER_CONF DEFAULT my_ip "$CINDER_SERVICE_HOST"

iniset $CINDER_CONF DEFAULT iscsi_helper tgtadm

iniset $CINDER_CONF DEFAULT sql_connection `database_connection_url cinder`

iniset $CINDER_CONF DEFAULT api_paste_config $CINDER_API_PASTE_INI

iniset $CINDER_CONF DEFAULT rootwrap_config "$CINDER_CONF_DIR/rootwrap.conf"

iniset $CINDER_CONF DEFAULT osapi_volume_extension cinder.api.contrib.standard_extensions

iniset $CINDER_CONF DEFAULT state_path $CINDER_STATE_PATH

iniset $CINDER_CONF DEFAULT lock_path $CINDER_STATE_PATH

iniset $CINDER_CONF DEFAULT periodic_interval $CINDER_PERIODIC_INTERVAL

# NOTE(thingee): Cinder V1 API is deprecated and defaults to off as of

# Juno. Keep it enabled so we can continue testing while it's still

# supported.

iniset $CINDER_CONF DEFAULT enable_v1_api true

if is_service_enabled c-vol && [[ -n "$CINDER_ENABLED_BACKENDS" ]]; then

local enabled_backends=""

local default_name=""

local be be_name be_type

for be in ${CINDER_ENABLED_BACKENDS//,/ }; do

be_type=${be%%:*}

be_name=${be##*:}

if type configure_cinder_backend_${be_type} >/dev/null 2>&1; then

configure_cinder_backend_${be_type} ${be_name}

fi

if [[ -z "$default_name" ]]; then

default_name=$be_name

fi

enabled_backends+=$be_name,

done

iniset $CINDER_CONF DEFAULT enabled_backends ${enabled_backends%,*}

if [[ -n "$default_name" ]]; then

iniset $CINDER_CONF DEFAULT default_volume_type ${default_name}

fi

fi

if is_service_enabled swift; then

iniset $CINDER_CONF DEFAULT backup_swift_url "http://$SERVICE_HOST:8080/v1/AUTH_"

fi

if is_service_enabled ceilometer; then

iniset $CINDER_CONF DEFAULT notification_driver "messaging"

fi

if is_service_enabled tls-proxy; then

# Set the service port for a proxy to take the original

iniset $CINDER_CONF DEFAULT osapi_volume_listen_port $CINDER_SERVICE_PORT_INT

fi

if [ "$SYSLOG" != "False" ]; then

iniset $CINDER_CONF DEFAULT use_syslog True

fi

iniset_rpc_backend cinder $CINDER_CONF DEFAULT

if [[ "$CINDER_SECURE_DELETE" == "False" ]]; then

iniset $CINDER_CONF DEFAULT secure_delete False

iniset $CINDER_CONF DEFAULT volume_clear none

fi

# Format logging

if [ "$LOG_COLOR" == "True" ] && [ "$SYSLOG" == "False" ]; then

setup_colorized_logging $CINDER_CONF DEFAULT "project_id" "user_id"

fi

if [[ -r $CINDER_PLUGINS/$CINDER_DRIVER ]]; then

configure_cinder_driver

fi

if [[ is_fedora && $DISTRO =~ (rhel6) ]]; then

# Cinder clones are slightly larger due to some extra

# metadata. RHEL6 will not allow auto-extending of LV's

# without this, leading to clones giving hard-to-track disk

# I/O errors.

# see https://bugzilla.redhat.com/show_bug.cgi?id=975052

sudo sed -i~ \

-e 's/snapshot_autoextend_threshold =.*/snapshot_autoextend_threshold = 80/' \

-e 's/snapshot_autoextend_percent =.*/snapshot_autoextend_percent = 20/' \

/etc/lvm/lvm.conf

fi

iniset $CINDER_CONF DEFAULT osapi_volume_workers "$API_WORKERS"

iniset $CINDER_CONF DEFAULT glance_api_servers "${GLANCE_SERVICE_PROTOCOL}://${GLANCE_HOSTPORT}"

if is_ssl_enabled_service glance || is_service_enabled tls-proxy; then

iniset $CINDER_CONF DEFAULT glance_protocol https

fi

# Register SSL certificates if provided

if is_ssl_enabled_service cinder; then

ensure_certificates CINDER

iniset $CINDER_CONF DEFAULT ssl_cert_file "$CINDER_SSL_CERT"

iniset $CINDER_CONF DEFAULT ssl_key_file "$CINDER_SSL_KEY"

fi

}

# create_cinder_accounts() - Set up common required cinder accounts

# Tenant User Roles

# ------------------------------------------------------------------

# service cinder admin # if enabled

# Migrated from keystone_data.sh

function create_cinder_accounts {

local service_tenant=$(openstack project list | awk "/ $SERVICE_TENANT_NAME / { print \$2 }")

local admin_role=$(openstack role list | awk "/ admin / { print \$2 }")

# Cinder

if [[ "$ENABLED_SERVICES" =~ "c-api" ]]; then

local cinder_user=$(get_or_create_user "cinder" \

"$SERVICE_PASSWORD" $service_tenant)

get_or_add_user_role $admin_role $cinder_user $service_tenant

if [[ "$KEYSTONE_CATALOG_BACKEND" = 'sql' ]]; then

local cinder_service=$(get_or_create_service "cinder" \

"volume" "Cinder Volume Service")

get_or_create_endpoint $cinder_service "$REGION_NAME" \

"$CINDER_SERVICE_PROTOCOL://$CINDER_SERVICE_HOST:$CINDER_SERVICE_PORT/v1/\$(tenant_id)s" \

"$CINDER_SERVICE_PROTOCOL://$CINDER_SERVICE_HOST:$CINDER_SERVICE_PORT/v1/\$(tenant_id)s" \

"$CINDER_SERVICE_PROTOCOL://$CINDER_SERVICE_HOST:$CINDER_SERVICE_PORT/v1/\$(tenant_id)s"

local cinder_v2_service=$(get_or_create_service "cinderv2" \

"volumev2" "Cinder Volume Service V2")

get_or_create_endpoint $cinder_v2_service "$REGION_NAME" \

"$CINDER_SERVICE_PROTOCOL://$CINDER_SERVICE_HOST:$CINDER_SERVICE_PORT/v2/\$(tenant_id)s" \

"$CINDER_SERVICE_PROTOCOL://$CINDER_SERVICE_HOST:$CINDER_SERVICE_PORT/v2/\$(tenant_id)s" \

"$CINDER_SERVICE_PROTOCOL://$CINDER_SERVICE_HOST:$CINDER_SERVICE_PORT/v2/\$(tenant_id)s"

fi

fi

}

# create_cinder_cache_dir() - Part of the init_cinder() process

function create_cinder_cache_dir {

# Create cache dir

sudo mkdir -p $CINDER_AUTH_CACHE_DIR

sudo chown $STACK_USER $CINDER_AUTH_CACHE_DIR

rm -f $CINDER_AUTH_CACHE_DIR/*

}

# init_cinder() - Initialize database and volume group

# Uses global ``NOVA_ENABLED_APIS``

function init_cinder {

# Force nova volumes off

NOVA_ENABLED_APIS=$(echo $NOVA_ENABLED_APIS | sed "s/osapi_volume,//")

if is_service_enabled $DATABASE_BACKENDS; then

# (Re)create cinder database

recreate_database cinder utf8

# Migrate cinder database

$CINDER_BIN_DIR/cinder-manage db sync

fi

if is_service_enabled c-vol && [[ -n "$CINDER_ENABLED_BACKENDS" ]]; then

local be be_name be_type

for be in ${CINDER_ENABLED_BACKENDS//,/ }; do

be_type=${be%%:*}

be_name=${be##*:}

if type init_cinder_backend_${be_type} >/dev/null 2>&1; then

init_cinder_backend_${be_type} ${be_name}

fi

done

fi

mkdir -p $CINDER_STATE_PATH/volumes

create_cinder_cache_dir

}

# install_cinder() - Collect source and prepare

function install_cinder {

git_clone $CINDER_REPO $CINDER_DIR $CINDER_BRANCH

setup_develop $CINDER_DIR

}

# install_cinderclient() - Collect source and prepare

function install_cinderclient {

git_clone $CINDERCLIENT_REPO $CINDERCLIENT_DIR $CINDERCLIENT_BRANCH

setup_develop $CINDERCLIENT_DIR

sudo install -D -m 0644 -o $STACK_USER {$CINDERCLIENT_DIR/tools/,/etc/bash_completion.d/}cinder.bash_completion

}

# apply config.d approach for cinder volumes directory

function _configure_tgt_for_config_d {

if [[ ! -d /etc/tgt/stack.d/ ]]; then

sudo ln -sf $CINDER_STATE_PATH/volumes /etc/tgt/stack.d

echo "include /etc/tgt/stack.d/*" | sudo tee -a /etc/tgt/targets.conf

fi

}

# start_cinder() - Start running processes, including screen

function start_cinder {

local service_port=$CINDER_SERVICE_PORT

local service_protocol=$CINDER_SERVICE_PROTOCOL

if is_service_enabled tls-proxy; then

service_port=$CINDER_SERVICE_PORT_INT

service_protocol="http"

fi

if is_service_enabled c-vol; then

# Delete any old stack.conf

sudo rm -f /etc/tgt/conf.d/stack.conf

_configure_tgt_for_config_d

if is_ubuntu; then

sudo service tgt restart

elif is_fedora; then

if [[ $DISTRO =~ (rhel6) ]]; then

sudo /sbin/service tgtd restart

else

# bypass redirection to systemctl during restart

sudo /sbin/service --skip-redirect tgtd restart

fi

elif is_suse; then

restart_service tgtd

else

# note for other distros: unstack.sh also uses the tgt/tgtd service

# name, and would need to be adjusted too

exit_distro_not_supported "restarting tgt"

fi

# NOTE(gfidente): ensure tgtd is running in debug mode

sudo tgtadm --mode system --op update --name debug --value on

fi

run_process c-api "$CINDER_BIN_DIR/cinder-api --config-file $CINDER_CONF"

echo "Waiting for Cinder API to start..."

if ! wait_for_service $SERVICE_TIMEOUT $service_protocol://$CINDER_SERVICE_HOST:$service_port; then

die $LINENO "c-api did not start"

fi

run_process c-sch "$CINDER_BIN_DIR/cinder-scheduler --config-file $CINDER_CONF"

run_process c-bak "$CINDER_BIN_DIR/cinder-backup --config-file $CINDER_CONF"

run_process c-vol "$CINDER_BIN_DIR/cinder-volume --config-file $CINDER_CONF"

# NOTE(jdg): For cinder, startup order matters. To ensure that repor_capabilities is received

# by the scheduler start the cinder-volume service last (or restart it) after the scheduler

# has started. This is a quick fix for lp bug/1189595

# Start proxies if enabled

if is_service_enabled c-api && is_service_enabled tls-proxy; then

start_tls_proxy '*' $CINDER_SERVICE_PORT $CINDER_SERVICE_HOST $CINDER_SERVICE_PORT_INT &

fi

}

# stop_cinder() - Stop running processes

function stop_cinder {

# Kill the cinder screen windows

local serv

for serv in c-api c-bak c-sch c-vol; do

stop_process $serv

done

if is_service_enabled c-vol; then

if is_ubuntu; then

stop_service tgt

else

stop_service tgtd

fi

fi

}

# create_volume_types() - Create Cinder's configured volume types

function create_volume_types {

# Create volume types

if is_service_enabled c-api && [[ -n "$CINDER_ENABLED_BACKENDS" ]]; then

local be be_name be_type

for be in ${CINDER_ENABLED_BACKENDS//,/ }; do

be_type=${be%%:*}

be_name=${be##*:}

# openstack volume type create --property volume_backend_name="${be_type}" ${be_name}

cinder type-create ${be_name} && \

cinder type-key ${be_name} set volume_backend_name="${be_name}"

done

fi

}

# Compatibility for Grenade

function create_cinder_volume_group {

# During a transition period Grenade needs to have this function defined

# It is effectively a no-op in the Grenade 'target' use case

:

}

# Restore xtrace

$XTRACE

# Tell emacs to use shell-script-mode

## Local variables:

## mode: shell-script

## End: