# functions - DevStack-specific functions

#

# The following variables are assumed to be defined by certain functions:

#

# - ``DATABASE_BACKENDS``

# - ``ENABLED_SERVICES``

# - ``FILES``

# - ``GLANCE_HOSTPORT``

#

# Include the common functions

FUNC_DIR=$(cd $(dirname "${BASH_SOURCE:-$0}") && pwd)

source ${FUNC_DIR}/functions-common

# Save trace setting

XTRACE=$(set +o | grep xtrace)

set +o xtrace

# Check if a function already exists

function function_exists {

declare -f -F $1 > /dev/null

}

# Retrieve an image from a URL and upload into Glance.

# Uses the following variables:

#

# - ``FILES`` must be set to the cache dir

# - ``GLANCE_HOSTPORT``

#

# upload_image image-url glance-token

function upload_image {

local image_url=$1

local token=$2

local image image_fname image_name

# Create a directory for the downloaded image tarballs.

mkdir -p $FILES/images

image_fname=`basename "$image_url"`

if [[ $image_url != file* ]]; then

# Downloads the image (uec ami+akistyle), then extracts it.

if [[ ! -f $FILES/$image_fname || "$(stat -c "%s" $FILES/$image_fname)" = "0" ]]; then

wget -c $image_url -O $FILES/$image_fname

if [[ $? -ne 0 ]]; then

echo "Not found: $image_url"

return

fi

fi

image="$FILES/${image_fname}"

else

# File based URL (RFC 1738): ``file://host/path``

# Remote files are not considered here.

# unix: ``file:///home/user/path/file``

# windows: ``file:///C:/Documents%20and%20Settings/user/path/file``

image=$(echo $image_url | sed "s/^file:\/\///g")

if [[ ! -f $image || "$(stat -c "%s" $image)" == "0" ]]; then

echo "Not found: $image_url"

return

fi

fi

# OpenVZ-format images are provided as .tar.gz, but not decompressed prior to loading

if [[ "$image_url" =~ 'openvz' ]]; then

image_name="${image_fname%.tar.gz}"

openstack --os-token $token --os-url $GLANCE_SERVICE_PROTOCOL://$GLANCE_HOSTPORT image create "$image_name" --public --container-format ami --disk-format ami < "${image}"

return

fi

# vmdk format images

if [[ "$image_url" =~ '.vmdk' ]]; then

image_name="${image_fname%.vmdk}"

# Before we can upload vmdk type images to glance, we need to know it's

# disk type, storage adapter, and networking adapter. These values are

# passed to glance as custom properties.

# We take these values from the vmdk file if populated. Otherwise, we use

# vmdk filename, which is expected in the following format:

#

# <name>-<disk type>;<storage adapter>;<network adapter>

#

# If the filename does not follow the above format then the vsphere

# driver will supply default values.

local vmdk_disktype=""

local vmdk_net_adapter=""

local path_len

# vmdk adapter type

local vmdk_adapter_type="$(head -25 $image | { grep -a -F -m 1 'ddb.adapterType =' $image || true; })"

vmdk_adapter_type="${vmdk_adapter_type#*\"}"

vmdk_adapter_type="${vmdk_adapter_type%?}"

# vmdk disk type

local vmdk_create_type="$(head -25 $image | { grep -a -F -m 1 'createType=' $image || true; })"

vmdk_create_type="${vmdk_create_type#*\"}"

vmdk_create_type="${vmdk_create_type%\"*}"

descriptor_data_pair_msg="Monolithic flat and VMFS disks "`

`"should use a descriptor-data pair."

if [[ "$vmdk_create_type" = "monolithicSparse" ]]; then

vmdk_disktype="sparse"

elif [[ "$vmdk_create_type" = "monolithicFlat" || "$vmdk_create_type" = "vmfs" ]]; then

# Attempt to retrieve the ``*-flat.vmdk``

local flat_fname="$(head -25 $image | { grep -G 'RW\|RDONLY [0-9]+ FLAT\|VMFS' $image || true; })"

flat_fname="${flat_fname#*\"}"

flat_fname="${flat_fname%?}"

if [[ -z "$flat_fname" ]]; then

flat_fname="$image_name-flat.vmdk"

fi

path_len=`expr ${#image_url} - ${#image_fname}`

local flat_url="${image_url:0:$path_len}$flat_fname"

warn $LINENO "$descriptor_data_pair_msg"`

`" Attempt to retrieve the *-flat.vmdk: $flat_url"

if [[ $flat_url != file* ]]; then

if [[ ! -f $FILES/$flat_fname || \

"$(stat -c "%s" $FILES/$flat_fname)" = "0" ]]; then

wget -c $flat_url -O $FILES/$flat_fname

fi

image="$FILES/${flat_fname}"

else

image=$(echo $flat_url | sed "s/^file:\/\///g")

if [[ ! -f $image || "$(stat -c "%s" $image)" == "0" ]]; then

echo "Flat disk not found: $flat_url"

return 1

fi

fi

image_name="${flat_fname}"

vmdk_disktype="preallocated"

elif [[ "$vmdk_create_type" = "streamOptimized" ]]; then

vmdk_disktype="streamOptimized"

elif [[ -z "$vmdk_create_type" ]]; then

# *-flat.vmdk provided: attempt to retrieve the descriptor (*.vmdk)

# to retrieve appropriate metadata

if [[ ${image_name: -5} != "-flat" ]]; then

warn $LINENO "Expected filename suffix: '-flat'."`

`" Filename provided: ${image_name}"

else

descriptor_fname="${image_name:0:${#image_name} - 5}.vmdk"

path_len=`expr ${#image_url} - ${#image_fname}`

local flat_path="${image_url:0:$path_len}"

local descriptor_url=$flat_path$descriptor_fname

warn $LINENO "$descriptor_data_pair_msg"`

`" Attempt to retrieve the descriptor *.vmdk: $descriptor_url"

if [[ $flat_path != file* ]]; then

if [[ ! -f $FILES/$descriptor_fname || \

"$(stat -c "%s" $FILES/$descriptor_fname)" = "0" ]]; then

wget -c $descriptor_url -O $FILES/$descriptor_fname

fi

descriptor_url="$FILES/$descriptor_fname"

else

descriptor_url=$(echo $descriptor_url | sed "s/^file:\/\///g")

if [[ ! -f $descriptor_url || \

"$(stat -c "%s" $descriptor_url)" == "0" ]]; then

echo "Descriptor not found: $descriptor_url"

return 1

fi

fi

vmdk_adapter_type="$(head -25 $descriptor_url | { grep -a -F -m 1 'ddb.adapterType =' $descriptor_url || true; })"

vmdk_adapter_type="${vmdk_adapter_type#*\"}"

vmdk_adapter_type="${vmdk_adapter_type%?}"

fi

vmdk_disktype="preallocated"

else

vmdk_disktype="preallocated"

fi

# NOTE: For backwards compatibility reasons, colons may be used in place

# of semi-colons for property delimiters but they are not permitted

# characters in NTFS filesystems.

property_string=`echo "$image_name" | { grep -oP '(?<=-)(?!.*-).*[:;].*[:;].*$' || true; }`

IFS=':;' read -a props <<< "$property_string"

vmdk_disktype="${props[0]:-$vmdk_disktype}"

vmdk_adapter_type="${props[1]:-$vmdk_adapter_type}"

vmdk_net_adapter="${props[2]:-$vmdk_net_adapter}"

openstack --os-token $token --os-url $GLANCE_SERVICE_PROTOCOL://$GLANCE_HOSTPORT image create "$image_name" --public --container-format bare --disk-format vmdk --property vmware_disktype="$vmdk_disktype" --property vmware_adaptertype="$vmdk_adapter_type" --property hw_vif_model="$vmdk_net_adapter" < "${image}"

return

fi

# XenServer-vhd-ovf-format images are provided as .vhd.tgz

# and should not be decompressed prior to loading

if [[ "$image_url" =~ '.vhd.tgz' ]]; then

image_name="${image_fname%.vhd.tgz}"

local force_vm_mode=""

if [[ "$image_name" =~ 'cirros' ]]; then

# Cirros VHD image currently only boots in PV mode.

# Nova defaults to PV for all VHD images, but

# the glance setting is needed for booting

# directly from volume.

force_vm_mode="--property vm_mode=xen"

fi

openstack \

--os-token $token \

--os-url $GLANCE_SERVICE_PROTOCOL://$GLANCE_HOSTPORT \

image create \

"$image_name" --public \

--container-format=ovf --disk-format=vhd \

$force_vm_mode < "${image}"

return

fi

# .xen-raw.tgz suggests a Xen capable raw image inside a tgz.

# and should not be decompressed prior to loading.

# Setting metadata, so PV mode is used.

if [[ "$image_url" =~ '.xen-raw.tgz' ]]; then

image_name="${image_fname%.xen-raw.tgz}"

openstack \

--os-token $token \

--os-url $GLANCE_SERVICE_PROTOCOL://$GLANCE_HOSTPORT \

image create \

"$image_name" --public \

--container-format=tgz --disk-format=raw \

--property vm_mode=xen < "${image}"

return

fi

local kernel=""

local ramdisk=""

local disk_format=""

local container_format=""

local unpack=""

local img_property=""

case "$image_fname" in

*.tar.gz|*.tgz)

# Extract ami and aki files

[ "${image_fname%.tar.gz}" != "$image_fname" ] &&

image_name="${image_fname%.tar.gz}" ||

image_name="${image_fname%.tgz}"

local xdir="$FILES/images/$image_name"

rm -Rf "$xdir";

mkdir "$xdir"

tar -zxf $image -C "$xdir"

kernel=$(for f in "$xdir/"*-vmlinuz* "$xdir/"aki-*/image; do

[ -f "$f" ] && echo "$f" && break; done; true)

ramdisk=$(for f in "$xdir/"*-initrd* "$xdir/"ari-*/image; do

[ -f "$f" ] && echo "$f" && break; done; true)

image=$(for f in "$xdir/"*.img "$xdir/"ami-*/image; do

[ -f "$f" ] && echo "$f" && break; done; true)

if [[ -z "$image_name" ]]; then

image_name=$(basename "$image" ".img")

fi

;;

*.img)

image_name=$(basename "$image" ".img")

local format=$(qemu-img info ${image} | awk '/^file format/ { print $3; exit }')

if [[ ",qcow2,raw,vdi,vmdk,vpc," =~ ",$format," ]]; then

disk_format=$format

else

disk_format=raw

fi

container_format=bare

;;

*.img.gz)

image_name=$(basename "$image" ".img.gz")

disk_format=raw

container_format=bare

unpack=zcat

;;

*.qcow2)

image_name=$(basename "$image" ".qcow2")

disk_format=qcow2

container_format=bare

;;

*.iso)

image_name=$(basename "$image" ".iso")

disk_format=iso

container_format=bare

;;

*.vhd|*.vhdx|*.vhd.gz|*.vhdx.gz)

local extension="${image_fname#*.}"

image_name=$(basename "$image" ".$extension")

disk_format=vhd

container_format=bare

if [ "${image_fname##*.}" == "gz" ]; then

unpack=zcat

fi

;;

*) echo "Do not know what to do with $image_fname"; false;;

esac

if is_arch "ppc64"; then

img_property="--property hw_cdrom_bus=scsi"

fi

if [ "$container_format" = "bare" ]; then

if [ "$unpack" = "zcat" ]; then

openstack --os-token $token --os-url $GLANCE_SERVICE_PROTOCOL://$GLANCE_HOSTPORT image create "$image_name" $img_property --public --container-format=$container_format --disk-format $disk_format < <(zcat --force "${image}")

else

openstack --os-token $token --os-url $GLANCE_SERVICE_PROTOCOL://$GLANCE_HOSTPORT image create "$image_name" $img_property --public --container-format=$container_format --disk-format $disk_format < "${image}"

fi

else

# Use glance client to add the kernel the root filesystem.

# We parse the results of the first upload to get the glance ID of the

# kernel for use when uploading the root filesystem.

local kernel_id="" ramdisk_id="";

if [ -n "$kernel" ]; then

kernel_id=$(openstack --os-token $token --os-url $GLANCE_SERVICE_PROTOCOL://$GLANCE_HOSTPORT image create "$image_name-kernel" $img_property --public --container-format aki --disk-format aki < "$kernel" | grep ' id ' | get_field 2)

fi

if [ -n "$ramdisk" ]; then

ramdisk_id=$(openstack --os-token $token --os-url $GLANCE_SERVICE_PROTOCOL://$GLANCE_HOSTPORT image create "$image_name-ramdisk" $img_property --public --container-format ari --disk-format ari < "$ramdisk" | grep ' id ' | get_field 2)

fi

openstack --os-token $token --os-url $GLANCE_SERVICE_PROTOCOL://$GLANCE_HOSTPORT image create "${image_name%.img}" $img_property --public --container-format ami --disk-format ami ${kernel_id:+--property kernel_id=$kernel_id} ${ramdisk_id:+--property ramdisk_id=$ramdisk_id} < "${image}"

fi

}

# Set the database backend to use

# When called from stackrc/localrc DATABASE_BACKENDS has not been

# initialized yet, just save the configuration selection and call back later

# to validate it.

#

# ``$1`` - the name of the database backend to use (mysql, postgresql, ...)

function use_database {

if [[ -z "$DATABASE_BACKENDS" ]]; then

# No backends registered means this is likely called from ``localrc``

# This is now deprecated usage

DATABASE_TYPE=$1

DEPRECATED_TEXT="$DEPRECATED_TEXT\nThe database backend needs to be properly set in ENABLED_SERVICES; use_database is deprecated localrc\n"

else

# This should no longer get called...here for posterity

use_exclusive_service DATABASE_BACKENDS DATABASE_TYPE $1

fi

}

# Wait for an HTTP server to start answering requests

# wait_for_service timeout url

function wait_for_service {

local timeout=$1

local url=$2

timeout $timeout sh -c "while ! curl -k --noproxy '*' -s $url >/dev/null; do sleep 1; done"

}

# ping check

# Uses globals ``ENABLED_SERVICES``

# ping_check from-net ip boot-timeout expected

function ping_check {

if is_service_enabled neutron; then

_ping_check_neutron "$1" $2 $3 $4

return

fi

_ping_check_novanet "$1" $2 $3 $4

}

# ping check for nova

# Uses globals ``MULTI_HOST``, ``PRIVATE_NETWORK``

function _ping_check_novanet {

local from_net=$1

local ip=$2

local boot_timeout=$3

local expected=${4:-"True"}

local check_command=""

MULTI_HOST=`trueorfalse False $MULTI_HOST`

if [[ "$MULTI_HOST" = "True" && "$from_net" = "$PRIVATE_NETWORK_NAME" ]]; then

return

fi

if [[ "$expected" = "True" ]]; then

check_command="while ! ping -c1 -w1 $ip; do sleep 1; done"

else

check_command="while ping -c1 -w1 $ip; do sleep 1; done"

fi

if ! timeout $boot_timeout sh -c "$check_command"; then

if [[ "$expected" = "True" ]]; then

die $LINENO "[Fail] Couldn't ping server"

else

die $LINENO "[Fail] Could ping server"

fi

fi

}

# Get ip of instance

function get_instance_ip {

local vm_id=$1

local network_name=$2

local nova_result="$(nova show $vm_id)"

local ip=$(echo "$nova_result" | grep "$network_name" | get_field 2)

if [[ $ip = "" ]];then

echo "$nova_result"

die $LINENO "[Fail] Coudn't get ipaddress of VM"

fi

echo $ip

}

# ssh check

# ssh_check net-name key-file floating-ip default-user active-timeout

function ssh_check {

if is_service_enabled neutron; then

_ssh_check_neutron "$1" $2 $3 $4 $5

return

fi

_ssh_check_novanet "$1" $2 $3 $4 $5

}

function _ssh_check_novanet {

local NET_NAME=$1

local KEY_FILE=$2

local FLOATING_IP=$3

local DEFAULT_INSTANCE_USER=$4

local ACTIVE_TIMEOUT=$5

local probe_cmd=""

if ! timeout $ACTIVE_TIMEOUT sh -c "while ! ssh -o StrictHostKeyChecking=no -i $KEY_FILE ${DEFAULT_INSTANCE_USER}@$FLOATING_IP echo success; do sleep 1; done"; then

die $LINENO "server didn't become ssh-able!"

fi

}

# Get the location of the $module-rootwrap executables, where module is cinder

# or nova.

# get_rootwrap_location module

function get_rootwrap_location {

local module=$1

echo "$(get_python_exec_prefix)/$module-rootwrap"

}

# Path permissions sanity check

# check_path_perm_sanity path

function check_path_perm_sanity {

# Ensure no element of the path has 0700 permissions, which is very

# likely to cause issues for daemons. Inspired by default 0700

# homedir permissions on RHEL and common practice of making DEST in

# the stack user's homedir.

local real_path=$(readlink -f $1)

local rebuilt_path=""

for i in $(echo ${real_path} | tr "/" " "); do

rebuilt_path=$rebuilt_path"/"$i

if [[ $(stat -c '%a' ${rebuilt_path}) = 700 ]]; then

echo "*** DEST path element"

echo "*** ${rebuilt_path}"

echo "*** appears to have 0700 permissions."

echo "*** This is very likely to cause fatal issues for devstack daemons."

if [[ -n "$SKIP_PATH_SANITY" ]]; then

return

else

echo "*** Set SKIP_PATH_SANITY to skip this check"

die $LINENO "Invalid path permissions"

fi

fi

done

}

# This function recursively compares versions, and is not meant to be

# called by anything other than vercmp_numbers below. This function does

# not work with alphabetic versions.

#

# _vercmp_r sep ver1 ver2

function _vercmp_r {

typeset sep

typeset -a ver1=() ver2=()

sep=$1; shift

ver1=("${@:1:sep}")

ver2=("${@:sep+1}")

if ((ver1 > ver2)); then

echo 1; return 0

elif ((ver2 > ver1)); then

echo -1; return 0

fi

if ((sep <= 1)); then

echo 0; return 0

fi

_vercmp_r $((sep-1)) "${ver1[@]:1}" "${ver2[@]:1}"

}

# This function compares two versions and is meant to be called by

# external callers. Please note the function assumes non-alphabetic

# versions. For example, this will work:

#

# vercmp_numbers 1.10 1.4

#

# The above will return "1", as 1.10 is greater than 1.4.

#

# vercmp_numbers 5.2 6.4

#

# The above will return "-1", as 5.2 is less than 6.4.

#

# vercmp_numbers 4.0 4.0

#

# The above will return "0", as the versions are equal.

#

# vercmp_numbers ver1 ver2

function vercmp_numbers {

typeset v1=$1 v2=$2 sep

typeset -a ver1 ver2

IFS=. read -ra ver1 <<< "$v1"

IFS=. read -ra ver2 <<< "$v2"

_vercmp_r "${#ver1[@]}" "${ver1[@]}" "${ver2[@]}"

}

# This function sets log formatting options for colorizing log

# output to stdout. It is meant to be called by lib modules.

# The last two parameters are optional and can be used to specify

# non-default value for project and user format variables.

# Defaults are respectively 'project_name' and 'user_name'

#

# setup_colorized_logging something.conf SOMESECTION

function setup_colorized_logging {

local conf_file=$1

local conf_section=$2

local project_var=${3:-"project_name"}

local user_var=${4:-"user_name"}

# Add color to logging output

iniset $conf_file $conf_section logging_context_format_string "%(asctime)s.%(msecs)03d %(color)s%(levelname)s %(name)s [%(request_id)s %("$user_var")s %("$project_var")s%(color)s] %(instance)s%(color)s%(message)s"

iniset $conf_file $conf_section logging_default_format_string "%(asctime)s.%(msecs)03d %(color)s%(levelname)s %(name)s [-%(color)s] %(instance)s%(color)s%(message)s"

iniset $conf_file $conf_section logging_debug_format_suffix "from (pid=%(process)d) %(funcName)s %(pathname)s:%(lineno)d"

iniset $conf_file $conf_section logging_exception_prefix "%(color)s%(asctime)s.%(msecs)03d TRACE %(name)s %(instance)s"

}

# These functions are provided for basic fall-back functionality for

# projects that include parts of devstack (grenade). stack.sh will

# override these with more specific versions for devstack (with fancy

# spinners, etc). We never override an existing version

if ! function_exists echo_summary; then

function echo_summary {

echo $@

}

fi

if ! function_exists echo_nolog; then

function echo_nolog {

echo $@

}

fi

# create_disk - Create backing disk

function create_disk {

local node_number

local disk_image=${1}

local storage_data_dir=${2}

local loopback_disk_size=${3}

# Create a loopback disk and format it to XFS.

if [[ -e ${disk_image} ]]; then

if egrep -q ${storage_data_dir} /proc/mounts; then

sudo umount ${storage_data_dir}/drives/sdb1

sudo rm -f ${disk_image}

fi

fi

sudo mkdir -p ${storage_data_dir}/drives/images

sudo truncate -s ${loopback_disk_size} ${disk_image}

# Make a fresh XFS filesystem. Use bigger inodes so xattr can fit in

# a single inode. Keeping the default inode size (256) will result in multiple

# inodes being used to store xattr. Retrieving the xattr will be slower

# since we have to read multiple inodes. This statement is true for both

# Swift and Ceph.

sudo mkfs.xfs -f -i size=1024 ${disk_image}

# Mount the disk with mount options to make it as efficient as possible

if ! egrep -q ${storage_data_dir} /proc/mounts; then

sudo mount -t xfs -o loop,noatime,nodiratime,nobarrier,logbufs=8 \

${disk_image} ${storage_data_dir}

fi

}

# Restore xtrace

$XTRACE

# Local variables:

# mode: shell-script

# End: