Skip to content

Commit d470867

Browse files
dgenindgenin
committed
Adds support for LVM ephemeral storage in Nova
DevStack currently lacks support for LVM ephemeral storage in Nova. This support is important for testing of Nova's LVM backend. The proposed change adds a default volume group, to be shared by Cinder and Nova. It also adds a configuration option NOVA_BACKEND, which must be LVM if it is set, that determines whether Nova should be configured to use LVM ephemeral storage. Change-Id: I4eb9afff3536fbcd563939f2d325efbb845081bb
1 parent 62002dd commit d470867

7 files changed

Lines changed: 150 additions & 125 deletions

File tree

‎HACKING.rst‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -110,8 +110,8 @@ follows:
110110
* Global configuration that may be referenced in ``local.conf``, i.e. ``DEST``, ``DATA_DIR``
111111
* Global service configuration like ``ENABLED_SERVICES``
112112
* Variables used by multiple services that do not have a clear owner, i.e.
113-
``VOLUME_BACKING_FILE_SIZE`` (nova-volumes and cinder) or ``PUBLIC_NETWORK_NAME``
114-
(nova-network and neutron)
113+
``VOLUME_BACKING_FILE_SIZE`` (nova-compute, nova-volumes and cinder) or
114+
``PUBLIC_NETWORK_NAME`` (nova-network and neutron)
115115
* Variables that can not be cleanly declared in a project file due to
116116
dependency ordering, i.e. the order of sourcing the project files can
117117
not be changed for other reasons but the earlier file needs to dereference a

‎lib/cinder‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -76,9 +76,9 @@ CINDER_MULTI_LVM_BACKEND=$(trueorfalse False CINDER_MULTI_LVM_BACKEND)
7676
# configuration and for the volume type name. Multiple backends are
7777
# comma-separated.
7878
if [[ $CINDER_MULTI_LVM_BACKEND == "False" ]]; then
79-
CINDER_ENABLED_BACKENDS=${CINDER_ENABLED_BACKENDS:-lvm:lvmdriver-1}
79+
CINDER_ENABLED_BACKENDS=${CINDER_ENABLED_BACKENDS:-lvm:${DEFAULT_VOLUME_GROUP_NAME##*-}}
8080
else
81-
CINDER_ENABLED_BACKENDS=${CINDER_ENABLED_BACKENDS:-lvm:lvmdriver-1,lvm:lvmdriver-2}
81+
CINDER_ENABLED_BACKENDS=${CINDER_ENABLED_BACKENDS:-lvm:${DEFAULT_VOLUME_GROUP_NAME##*-},lvm:cinder}
8282
fi
8383

8484

‎lib/cinder_backends/lvm‎

Lines changed: 9 additions & 121 deletions
Original file line numberDiff line numberDiff line change
@@ -14,6 +14,7 @@
1414

1515
# CINDER_CONF
1616
# DATA_DIR
17+
# VOLUME_GROUP_NAME
1718

1819
# clean_cinder_backend_lvm - called from clean_cinder()
1920
# configure_cinder_backend_lvm - called from configure_cinder()
@@ -25,157 +26,44 @@ MY_XTRACE=$(set +o | grep xtrace)
2526
set +o xtrace
2627

2728

28-
# Defaults
29-
# --------
30-
31-
# Name of the lvm volume groups to use/create for iscsi volumes
32-
# This monkey-motion is for compatibility with icehouse-generation Grenade
33-
# If ``VOLUME_GROUP`` is set, use it, otherwise we'll build a VG name based
34-
# on ``VOLUME_GROUP_NAME`` that includes the backend name
35-
# Grenade doesn't use ``VOLUME_GROUP2`` so it is left out
36-
VOLUME_GROUP=${VOLUME_GROUP:-}
37-
VOLUME_GROUP_NAME=${VOLUME_GROUP:-${VOLUME_GROUP_NAME:-stack-volumes}}
38-
3929
# TODO: resurrect backing device...need to know how to set values
4030
#VOLUME_BACKING_DEVICE=${VOLUME_BACKING_DEVICE:-}
4131

42-
VOLUME_NAME_PREFIX=${VOLUME_NAME_PREFIX:-volume-}
43-
44-
4532
# Entry Points
4633
# ------------
4734

48-
# Compatibility for getting a volume group name from either ``VOLUME_GROUP``
49-
# or from ``VOLUME_GROUP_NAME`` plus the backend name
50-
function get_volume_group_name {
51-
local be_name=$1
52-
53-
# Again with the icehouse-generation compatibility
54-
local volume_group_name=$VOLUME_GROUP_NAME
55-
if [[ -z $VOLUME_GROUP ]]; then
56-
volume_group_name+="-$be_name"
57-
fi
58-
echo $volume_group_name
59-
}
60-
35+
# cleanup_cinder_backend_lvm - Delete volume group and remove backing file
36+
# cleanup_cinder_backend_lvm $be_name
6137
function cleanup_cinder_backend_lvm {
6238
local be_name=$1
6339

64-
# Again with the icehouse-generation compatibility
65-
local volume_group_name=$(get_volume_group_name $be_name)
66-
6740
# Campsite rule: leave behind a volume group at least as clean as we found it
68-
_clean_lvm_lv ${volume_group_name} $VOLUME_NAME_PREFIX
69-
_clean_lvm_backing_file ${volume_group_name} $DATA_DIR/${volume_group_name}-backing-file
41+
clean_lvm_volume_group $VOLUME_GROUP_NAME-$be_name
7042
}
7143

7244
# configure_cinder_backend_lvm - Set config files, create data dirs, etc
73-
# configure_cinder_backend_lvm $name
45+
# configure_cinder_backend_lvm $be_name
7446
function configure_cinder_backend_lvm {
7547
local be_name=$1
7648

77-
# Again with the icehouse-generation compatibility
78-
local volume_group_name=$(get_volume_group_name $be_name)
79-
8049
iniset $CINDER_CONF $be_name volume_backend_name $be_name
8150
iniset $CINDER_CONF $be_name volume_driver "cinder.volume.drivers.lvm.LVMISCSIDriver"
82-
iniset $CINDER_CONF $be_name volume_group $volume_group_name
51+
iniset $CINDER_CONF $be_name volume_group $VOLUME_GROUP_NAME-$be_name
8352

8453
if [[ "$CINDER_SECURE_DELETE" == "False" ]]; then
8554
iniset $CINDER_CONF $be_name volume_clear none
8655
fi
8756
}
8857

89-
58+
# init_cinder_backend_lvm - Initialize volume group
59+
# init_cinder_backend_lvm $be_name
9060
function init_cinder_backend_lvm {
9161
local be_name=$1
9262

93-
# Again with the icehouse-generation compatibility
94-
local volume_group_name=$(get_volume_group_name $be_name)
95-
9663
# Start with a clean volume group
97-
_create_cinder_volume_group ${volume_group_name} $DATA_DIR/${volume_group_name}-backing-file
98-
99-
if is_fedora || is_suse; then
100-
# service is not started by default
101-
start_service tgtd
102-
fi
103-
104-
# Remove iscsi targets
105-
sudo tgtadm --op show --mode target | grep $VOLUME_NAME_PREFIX | grep Target | cut -f3 -d ' ' | sudo xargs -n1 tgt-admin --delete || true
106-
_clean_lvm_lv ${volume_group_name} $VOLUME_NAME_PREFIX
107-
}
108-
109-
110-
# _clean_lvm_lv removes all cinder LVM volumes
111-
#
112-
# Usage: _clean_lvm_lv volume-group-name $VOLUME_NAME_PREFIX
113-
function _clean_lvm_lv {
114-
local vg=$1
115-
local lv_prefix=$2
116-
117-
# Clean out existing volumes
118-
local lv
119-
for lv in $(sudo lvs --noheadings -o lv_name $vg 2>/dev/null); do
120-
# lv_prefix prefixes the LVs we want
121-
if [[ "${lv#$lv_prefix}" != "$lv" ]]; then
122-
sudo lvremove -f $vg/$lv
123-
fi
124-
done
125-
}
126-
127-
# _clean_lvm_backing_file() removes the backing file of the
128-
# volume group used by cinder
129-
#
130-
# Usage: _clean_lvm_backing_file() volume-group-name backing-file-name
131-
function _clean_lvm_backing_file {
132-
local vg=$1
133-
local backing_file=$2
134-
135-
# if there is no logical volume left, it's safe to attempt a cleanup
136-
# of the backing file
137-
if [[ -z "$(sudo lvs --noheadings -o lv_name $vg 2>/dev/null)" ]]; then
138-
# if the backing physical device is a loop device, it was probably setup by devstack
139-
local vg_dev=$(sudo losetup -j $backing_file | awk -F':' '/backing-file/ { print $1}')
140-
if [[ -n "$vg_dev" ]] && [[ -e "$vg_dev" ]]; then
141-
sudo losetup -d $vg_dev
142-
rm -f $backing_file
143-
fi
144-
fi
64+
init_lvm_volume_group $VOLUME_GROUP_NAME-$be_name $VOLUME_BACKING_FILE_SIZE
14565
}
14666

147-
# _create_cinder_volume_group volume-group-name backing-file-name
148-
function _create_cinder_volume_group {
149-
# According to the ``CINDER_MULTI_LVM_BACKEND`` value, configure one or two default volumes
150-
# group called ``stack-volumes`` (and ``stack-volumes2``) for the volume
151-
# service if it (they) does (do) not yet exist. If you don't wish to use a
152-
# file backed volume group, create your own volume group called ``stack-volumes``
153-
# and ``stack-volumes2`` before invoking ``stack.sh``.
154-
#
155-
# The two backing files are ``VOLUME_BACKING_FILE_SIZE`` in size, and they are stored in
156-
# the ``DATA_DIR``.
157-
158-
local vg_name=$1
159-
local backing_file=$2
160-
161-
if ! sudo vgs $vg_name; then
162-
# TODO: fix device handling
163-
if [ -z "$VOLUME_BACKING_DEVICE" ]; then
164-
# Only create if the file doesn't already exists
165-
[[ -f $backing_file ]] || truncate -s $VOLUME_BACKING_FILE_SIZE $backing_file
166-
local vg_dev=`sudo losetup -f --show $backing_file`
167-
168-
# Only create if the loopback device doesn't contain $VOLUME_GROUP
169-
if ! sudo vgs $vg_name; then
170-
sudo vgcreate $vg_name $vg_dev
171-
fi
172-
else
173-
sudo vgcreate $vg_name $VOLUME_BACKING_DEVICE
174-
fi
175-
fi
176-
}
177-
178-
17967
# Restore xtrace
18068
$MY_XTRACE
18169

‎lib/lvm‎

Lines changed: 124 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,124 @@
1+
# lib/lvm
2+
# Configure the default LVM volume group used by Cinder and Nova
3+
4+
# Dependencies:
5+
#
6+
# - ``functions`` file
7+
# - ``cinder`` configurations
8+
9+
# DATA_DIR
10+
11+
# clean_default_volume_group - called from clean()
12+
# configure_default_volume_group - called from configure()
13+
# init_default_volume_group - called from init()
14+
15+
16+
# Save trace setting
17+
MY_XTRACE=$(set +o | grep xtrace)
18+
set +o xtrace
19+
20+
21+
# Defaults
22+
# --------
23+
# Name of the lvm volume groups to use/create for iscsi volumes
24+
# This monkey-motion is for compatibility with icehouse-generation Grenade
25+
# If ``VOLUME_GROUP`` is set, use it, otherwise we'll build a VG name based
26+
# on ``VOLUME_GROUP_NAME`` that includes the backend name
27+
# Grenade doesn't use ``VOLUME_GROUP2`` so it is left out
28+
VOLUME_GROUP_NAME=${VOLUME_GROUP:-${VOLUME_GROUP_NAME:-stack-volumes}}
29+
DEFAULT_VOLUME_GROUP_NAME=$VOLUME_GROUP_NAME-default
30+
31+
# Backing file name is of the form $VOLUME_GROUP$BACKING_FILE_SUFFIX
32+
BACKING_FILE_SUFFIX=-backing-file
33+
34+
35+
# Entry Points
36+
# ------------
37+
38+
# _clean_lvm_volume_group removes all default LVM volumes
39+
#
40+
# Usage: clean_lvm_volume_group $vg
41+
function _clean_lvm_volume_group {
42+
local vg=$1
43+
44+
# Clean out existing volumes
45+
sudo lvremove -f $vg
46+
}
47+
48+
# _clean_lvm_backing_file() removes the backing file of the
49+
# volume group
50+
#
51+
# Usage: _clean_lvm_backing_file() $backing_file
52+
function _clean_lvm_backing_file {
53+
local backing_file=$1
54+
55+
# if the backing physical device is a loop device, it was probably setup by devstack
56+
if [[ -n "$backing_file" ]] && [[ -e "$backing_file" ]]; then
57+
local vg_dev=$(sudo losetup -j $backing_file | awk -F':' '/'$BACKING_FILE_SUFFIX'/ { print $1}')
58+
sudo losetup -d $vg_dev
59+
rm -f $backing_file
60+
fi
61+
}
62+
63+
# clean_lvm_volume_group() cleans up the volume group and removes the
64+
# backing file
65+
#
66+
# Usage: clean_lvm_volume_group $vg
67+
function clean_lvm_volume_group {
68+
local vg=$1
69+
70+
_clean_lvm_volume_group $vg
71+
# if there is no logical volume left, it's safe to attempt a cleanup
72+
# of the backing file
73+
if [[ -z "$(sudo lvs --noheadings -o lv_name $vg 2>/dev/null)" ]]; then
74+
_clean_lvm_backing_file $DATA_DIR/$vg$BACKING_FILE_SUFFIX
75+
fi
76+
}
77+
78+
79+
# _create_volume_group creates default volume group
80+
#
81+
# Usage: _create_lvm_volume_group() $vg $size
82+
function _create_lvm_volume_group {
83+
local vg=$1
84+
local size=$2
85+
86+
local backing_file=$DATA_DIR/$vg$BACKING_FILE_SUFFIX
87+
if ! sudo vgs $vg; then
88+
# Only create if the file doesn't already exists
89+
[[ -f $DATA_DIR/$backing_file ]] || truncate -s $size $backing_file
90+
local vg_dev=`sudo losetup -f --show $backing_file`
91+
92+
# Only create volume group if it doesn't already exist
93+
if ! sudo vgs $vg; then
94+
sudo vgcreate $vg $vg_dev
95+
fi
96+
fi
97+
}
98+
99+
# init_lvm_volume_group() initializes the volume group creating the backing
100+
# file if necessary
101+
#
102+
# Usage: init_lvm_volume_group() $vg
103+
function init_lvm_volume_group {
104+
local vg=$1
105+
local size=$2
106+
# Start with a clean volume group
107+
_create_lvm_volume_group $vg $size
108+
109+
if is_fedora || is_suse; then
110+
# service is not started by default
111+
start_service tgtd
112+
fi
113+
114+
# Remove iscsi targets
115+
sudo tgtadm --op show --mode target | grep Target | cut -f3 -d ' ' | sudo xargs -n1 tgt-admin --delete || true
116+
117+
_clean_lvm_volume_group $vg
118+
}
119+
120+
# Restore xtrace
121+
$MY_XTRACE
122+
123+
# mode: shell-script
124+
# End:

‎lib/nova‎

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -547,6 +547,11 @@ function create_nova_conf {
547547
iniset $NOVA_CONF DEFAULT ec2_workers "$API_WORKERS"
548548
iniset $NOVA_CONF DEFAULT metadata_workers "$API_WORKERS"
549549

550+
if [[ "$NOVA_BACKEND" == "LVM" ]]; then
551+
iniset $NOVA_CONF libvirt images_type "lvm"
552+
iniset $NOVA_CONF libvirt images_volume_group $DEFAULT_VOLUME_GROUP_NAME
553+
fi
554+
550555
if is_ssl_enabled_service glance || is_service_enabled tls-proxy; then
551556
iniset $NOVA_CONF DEFAULT glance_protocol https
552557
fi

‎stack.sh‎

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -523,6 +523,7 @@ source $TOP_DIR/lib/tls
523523
source $TOP_DIR/lib/infra
524524
source $TOP_DIR/lib/oslo
525525
source $TOP_DIR/lib/stackforge
526+
source $TOP_DIR/lib/lvm
526527
source $TOP_DIR/lib/horizon
527528
source $TOP_DIR/lib/keystone
528529
source $TOP_DIR/lib/glance
@@ -939,6 +940,10 @@ init_service_check
939940
# A better kind of sysstat, with the top process per time slice
940941
start_dstat
941942

943+
# Initialize default LVM volume group
944+
# -----------------------------------
945+
init_lvm_volume_group $DEFAULT_VOLUME_GROUP_NAME $VOLUME_BACKING_FILE_SIZE
946+
942947
# Start Services
943948
# ==============
944949

‎unstack.sh‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -55,6 +55,7 @@ source $TOP_DIR/lib/tls
5555
source $TOP_DIR/lib/infra
5656
source $TOP_DIR/lib/oslo
5757
source $TOP_DIR/lib/stackforge
58+
source $TOP_DIR/lib/lvm
5859
source $TOP_DIR/lib/horizon
5960
source $TOP_DIR/lib/keystone
6061
source $TOP_DIR/lib/glance
@@ -177,3 +178,5 @@ if [[ -n "$SCREEN" ]]; then
177178
screen -X -S $SESSION quit
178179
fi
179180
fi
181+
182+
clean_lvm_volume_group $DEFAULT_VOLUME_GROUP_NAME

0 commit comments

Comments
 (0)