Skip to content

Commit 06fb29c

Browse files
Alexander Gordeevgandelman-a
authored andcommitted
Integration testing preparation for Ironic
Add ability to create/register qemu vms for Ironic testing purposes Implements bp:deprecate-baremetal-driver Change-Id: If452438fcc0ff562531b33a36cd189b235654b48
1 parent 1fef36e commit 06fb29c

File tree

15 files changed

+630
-8
lines changed

15 files changed

+630
-8
lines changed

extras.d/50-ironic.sh

Lines changed: 7 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -24,10 +24,17 @@ if is_service_enabled ir-api ir-cond; then
2424
# Start the ironic API and ironic taskmgr components
2525
echo_summary "Starting Ironic"
2626
start_ironic
27+
28+
if [[ "$IRONIC_BAREMETAL_BASIC_OPS" = "True" ]]; then
29+
prepare_baremetal_basic_ops
30+
fi
2731
fi
2832

2933
if [[ "$1" == "unstack" ]]; then
3034
stop_ironic
35+
if [[ "$IRONIC_BAREMETAL_BASIC_OPS" = "True" ]]; then
36+
cleanup_baremetal_basic_ops
37+
fi
3138
fi
3239

3340
if [[ "$1" == "clean" ]]; then

files/apts/ironic

Lines changed: 10 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,10 @@
1+
libguestfs0
2+
libvirt-bin
3+
openssh-client
4+
openvswitch-switch
5+
openvswitch-datapath-dkms
6+
python-libguestfs
7+
python-libvirt
8+
syslinux
9+
tftpd-hpa
10+
xinetd

files/rpms/ironic

Lines changed: 9 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,9 @@
1+
libguestfs
2+
libvirt
3+
libvirt-python
4+
openssh-clients
5+
openvswitch
6+
python-libguestfs
7+
syslinux
8+
tftp-server
9+
xinetd

lib/baremetal

Lines changed: 5 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -140,7 +140,10 @@ BM_DEPLOY_KERNEL=${BM_DEPLOY_KERNEL:-}
140140

141141
# If you need to add any extra flavors to the deploy ramdisk image
142142
# eg, specific network drivers, specify them here
143-
BM_DEPLOY_FLAVOR=${BM_DEPLOY_FLAVOR:-}
143+
#
144+
# NOTE(deva): this will be moved to lib/ironic in a future patch
145+
# for now, set the default to a suitable value for Ironic's needs
146+
BM_DEPLOY_FLAVOR=${BM_DEPLOY_FLAVOR:--a amd64 ubuntu deploy-ironic}
144147

145148
# set URL and version for google shell-in-a-box
146149
BM_SHELL_IN_A_BOX=${BM_SHELL_IN_A_BOX:-http://shellinabox.googlecode.com/files/shellinabox-2.14.tar.gz}
@@ -220,7 +223,7 @@ function upload_baremetal_deploy {
220223
BM_DEPLOY_KERNEL=bm-deploy.kernel
221224
BM_DEPLOY_RAMDISK=bm-deploy.initramfs
222225
if [ ! -e "$TOP_DIR/files/$BM_DEPLOY_KERNEL" -o ! -e "$TOP_DIR/files/$BM_DEPLOY_RAMDISK" ]; then
223-
$BM_IMAGE_BUILD_DIR/bin/ramdisk-image-create $BM_DEPLOY_FLAVOR deploy \
226+
$BM_IMAGE_BUILD_DIR/bin/ramdisk-image-create $BM_DEPLOY_FLAVOR \
224227
-o $TOP_DIR/files/bm-deploy
225228
fi
226229
fi

lib/ironic

Lines changed: 261 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -18,23 +18,48 @@
1818
# - stop_ironic
1919
# - cleanup_ironic
2020

21-
# Save trace setting
21+
# Save trace and pipefail settings
2222
XTRACE=$(set +o | grep xtrace)
23+
PIPEFAIL=$(set +o | grep pipefail)
2324
set +o xtrace
24-
25+
set +o pipefail
2526

2627
# Defaults
2728
# --------
2829

2930
# Set up default directories
3031
IRONIC_DIR=$DEST/ironic
32+
IRONIC_DATA_DIR=$DATA_DIR/ironic
33+
IRONIC_STATE_PATH=/var/lib/ironic
3134
IRONICCLIENT_DIR=$DEST/python-ironicclient
3235
IRONIC_AUTH_CACHE_DIR=${IRONIC_AUTH_CACHE_DIR:-/var/cache/ironic}
3336
IRONIC_CONF_DIR=${IRONIC_CONF_DIR:-/etc/ironic}
3437
IRONIC_CONF_FILE=$IRONIC_CONF_DIR/ironic.conf
3538
IRONIC_ROOTWRAP_CONF=$IRONIC_CONF_DIR/rootwrap.conf
3639
IRONIC_POLICY_JSON=$IRONIC_CONF_DIR/policy.json
3740

41+
# Set up defaults for functional / integration testing
42+
IRONIC_SCRIPTS_DIR=${IRONIC_SCRIPTS_DIR:-$TOP_DIR/tools/ironic/scripts}
43+
IRONIC_TEMPLATES_DIR=${IRONIC_TEMPLATES_DIR:-$TOP_DIR/tools/ironic/templates}
44+
IRONIC_BAREMETAL_BASIC_OPS=$(trueorfalse False $IRONIC_BAREMETAL_BASIC_OPS)
45+
IRONIC_SSH_USERNAME=${IRONIC_SSH_USERNAME:-`whoami`}
46+
IRONIC_SSH_KEY_DIR=${IRONIC_SSH_KEY_DIR:-$IRONIC_DATA_DIR/ssh_keys}
47+
IRONIC_SSH_KEY_FILENAME=${IRONIC_SSH_KEY_FILENAME:-ironic_key}
48+
IRONIC_KEY_FILE=$IRONIC_SSH_KEY_DIR/$IRONIC_SSH_KEY_FILENAME
49+
IRONIC_SSH_VIRT_TYPE=${IRONIC_SSH_VIRT_TYPE:-virsh}
50+
IRONIC_TFTPBOOT_DIR=${IRONIC_TFTPBOOT_DIR:-$IRONIC_DATA_DIR/tftpboot}
51+
IRONIC_VM_SSH_PORT=${IRONIC_VM_SSH_PORT:-2222}
52+
IRONIC_VM_SSH_ADDRESS=${IRONIC_VM_SSH_ADDRESS:-$HOST_IP}
53+
IRONIC_VM_COUNT=${IRONIC_VM_COUNT:-1}
54+
IRONIC_VM_SPECS_CPU=${IRONIC_VM_SPECS_CPU:-1}
55+
IRONIC_VM_SPECS_RAM=${IRONIC_VM_SPECS_RAM:-256}
56+
IRONIC_VM_SPECS_DISK=${IRONIC_VM_SPECS_DISK:-10}
57+
IRONIC_VM_EMULATOR=${IRONIC_VM_EMULATOR:-/usr/bin/qemu-system-x86_64}
58+
IRONIC_VM_NETWORK_BRIDGE=${IRONIC_VM_NETWORK_BRIDGE:-brbm}
59+
IRONIC_VM_NETWORK_RANGE=${IRONIC_VM_NETWORK_RANGE:-192.0.2.0/24}
60+
IRONIC_VM_MACS_CSV_FILE=${IRONIC_VM_MACS_CSV_FILE:-$IRONIC_DATA_DIR/ironic_macs.csv}
61+
IRONIC_AUTHORIZED_KEYS_FILE=${IRONIC_AUTHORIZED_KEYS_FILE:-$HOME/.ssh/authorized_keys}
62+
3863
# Support entry points installation of console scripts
3964
IRONIC_BIN_DIR=$(get_python_exec_prefix)
4065

@@ -86,8 +111,8 @@ function configure_ironic {
86111
iniset $IRONIC_CONF_FILE DEFAULT debug True
87112
inicomment $IRONIC_CONF_FILE DEFAULT log_file
88113
iniset $IRONIC_CONF_FILE DEFAULT sql_connection `database_connection_url ironic`
114+
iniset $IRONIC_CONF_FILE DEFAULT state_path $IRONIC_STATE_PATH
89115
iniset $IRONIC_CONF_FILE DEFAULT use_syslog $SYSLOG
90-
91116
# Configure Ironic conductor, if it was enabled.
92117
if is_service_enabled ir-cond; then
93118
configure_ironic_conductor
@@ -97,6 +122,10 @@ function configure_ironic {
97122
if is_service_enabled ir-api; then
98123
configure_ironic_api
99124
fi
125+
126+
if [[ "$IRONIC_BAREMETAL_BASIC_OPS" == "True" ]]; then
127+
configure_ironic_auxiliary
128+
fi
100129
}
101130

102131
# configure_ironic_api() - Is used by configure_ironic(). Performs
@@ -125,6 +154,10 @@ function configure_ironic_conductor {
125154
cp -r $IRONIC_DIR/etc/ironic/rootwrap.d $IRONIC_CONF_DIR
126155

127156
iniset $IRONIC_CONF_FILE DEFAULT rootwrap_config $IRONIC_ROOTWRAP_CONF
157+
iniset $IRONIC_CONF_FILE conductor api_url http://$SERVICE_HOST:6385
158+
iniset $IRONIC_CONF_FILE pxe tftp_server $SERVICE_HOST
159+
iniset $IRONIC_CONF_FILE pxe tftp_root $IRONIC_TFTPBOOT_DIR
160+
iniset $IRONIC_CONF_FILE pxe tftp_master_path $IRONIC_TFTPBOOT_DIR/master_images
128161
}
129162

130163
# create_ironic_cache_dir() - Part of the init_ironic() process
@@ -225,9 +258,233 @@ function stop_ironic {
225258
screen -S $SCREEN_NAME -p ir-cond -X kill
226259
}
227260

261+
function is_ironic {
262+
if ( is_service_enabled ir-cond && is_service_enabled ir-api ); then
263+
return 0
264+
fi
265+
return 1
266+
}
267+
268+
function configure_ironic_dirs {
269+
sudo mkdir -p $IRONIC_DATA_DIR
270+
sudo mkdir -p $IRONIC_STATE_PATH
271+
sudo mkdir -p $IRONIC_TFTPBOOT_DIR
272+
sudo chown -R $STACK_USER $IRONIC_DATA_DIR $IRONIC_STATE_PATH
273+
sudo chown -R $STACK_USER:$LIBVIRT_GROUP $IRONIC_TFTPBOOT_DIR
274+
if is_ubuntu; then
275+
PXEBIN=/usr/lib/syslinux/pxelinux.0
276+
elif is_fedora; then
277+
PXEBIN=/usr/share/syslinux/pxelinux.0
278+
fi
279+
if [ ! -f $PXEBIN ]; then
280+
die $LINENO "pxelinux.0 (from SYSLINUX) not found."
281+
fi
282+
283+
cp $PXEBIN $IRONIC_TFTPBOOT_DIR
284+
mkdir -p $IRONIC_TFTPBOOT_DIR/pxelinux.cfg
285+
}
286+
287+
function ironic_ensure_libvirt_group {
288+
groups $STACK_USER | grep -q $LIBVIRT_GROUP || adduser $STACK_USER $LIBVIRT_GROUP
289+
}
290+
291+
function create_bridge_and_vms {
292+
ironic_ensure_libvirt_group
293+
294+
# Call libvirt setup scripts in a new shell to ensure any new group membership
295+
sudo su $STACK_USER -c "$IRONIC_SCRIPTS_DIR/setup-network"
296+
297+
sudo su $STACK_USER -c "$IRONIC_SCRIPTS_DIR/create-nodes \
298+
$IRONIC_VM_SPECS_CPU $IRONIC_VM_SPECS_RAM $IRONIC_VM_SPECS_DISK \
299+
amd64 $IRONIC_VM_COUNT $IRONIC_VM_NETWORK_BRIDGE $IRONIC_VM_EMULATOR" >> $IRONIC_VM_MACS_CSV_FILE
300+
301+
}
302+
303+
function enroll_vms {
304+
305+
CHASSIS_ID=$(ironic chassis-create -d "ironic test chassis" | grep " uuid " | get_field 2)
306+
IRONIC_NET_ID=$(neutron net-list | grep private | get_field 1)
307+
local idx=0
308+
309+
# work around; need to know what netns neutron uses for private network
310+
neutron port-create private
311+
312+
while read MAC; do
313+
314+
NODE_ID=$(ironic node-create --chassis_uuid $CHASSIS_ID --driver pxe_ssh \
315+
-i ssh_virt_type=$IRONIC_SSH_VIRT_TYPE \
316+
-i ssh_address=$IRONIC_VM_SSH_ADDRESS \
317+
-i ssh_port=$IRONIC_VM_SSH_PORT \
318+
-i ssh_username=$IRONIC_SSH_USERNAME \
319+
-i ssh_key_filename=$IRONIC_SSH_KEY_DIR/$IRONIC_SSH_KEY_FILENAME \
320+
-p cpus=$IRONIC_VM_SPECS_CPU \
321+
-p memory_mb=$IRONIC_VM_SPECS_RAM \
322+
-p local_gb=$IRONIC_VM_SPECS_DISK \
323+
-p cpu_arch=x86_64 \
324+
| grep " uuid " | get_field 2)
325+
326+
ironic port-create --address $MAC --node_uuid $NODE_ID
327+
328+
idx=$((idx+1))
329+
330+
done < $IRONIC_VM_MACS_CSV_FILE
331+
332+
# create the nova flavor
333+
nova flavor-create baremetal auto $IRONIC_VM_SPECS_RAM $IRONIC_VM_SPECS_DISK $IRONIC_VM_SPECS_CPU
334+
nova flavor-key baremetal set "cpu_arch"="x86_64" "baremetal:deploy_kernel_id"="$BM_DEPLOY_KERNEL_ID" "baremetal:deploy_ramdisk_id"="$BM_DEPLOY_RAMDISK_ID"
335+
336+
# intentional sleep to make sure the tag has been set to port
337+
sleep 10
338+
TAPDEV=$(sudo ip netns exec qdhcp-${IRONIC_NET_ID} ip link list | grep tap | cut -d':' -f2 | cut -b2-)
339+
TAG_ID=$(sudo ovs-vsctl show |grep ${TAPDEV} -A1 -m1 | grep tag | cut -d':' -f2 | cut -b2-)
340+
341+
# make sure veth pair is not existing, otherwise delete its links
342+
sudo ip link show ovs-tap1 && sudo ip link delete ovs-tap1
343+
sudo ip link show brbm-tap1 && sudo ip link delete brbm-tap1
344+
# create veth pair for future interconnection between br-int and brbm
345+
sudo ip link add brbm-tap1 type veth peer name ovs-tap1
346+
sudo ip link set dev brbm-tap1 up
347+
sudo ip link set dev ovs-tap1 up
348+
349+
sudo ovs-vsctl -- --if-exists del-port ovs-tap1 -- add-port br-int ovs-tap1 tag=$TAG_ID
350+
sudo ovs-vsctl -- --if-exists del-port brbm-tap1 -- add-port $IRONIC_VM_NETWORK_BRIDGE brbm-tap1
351+
}
352+
353+
function configure_tftpd {
354+
# enable tftp natting for allowing connections to SERVICE_HOST's tftp server
355+
sudo modprobe nf_conntrack_tftp
356+
sudo modprobe nf_nat_tftp
357+
358+
if is_ubuntu; then
359+
PXEBIN=/usr/lib/syslinux/pxelinux.0
360+
elif is_fedora; then
361+
PXEBIN=/usr/share/syslinux/pxelinux.0
362+
fi
363+
if [ ! -f $PXEBIN ]; then
364+
die $LINENO "pxelinux.0 (from SYSLINUX) not found."
365+
fi
366+
367+
# stop tftpd and setup serving via xinetd
368+
stop_service tftpd-hpa || true
369+
[ -f /etc/init/tftpd-hpa.conf ] && echo "manual" | sudo tee /etc/init/tftpd-hpa.override
370+
sudo cp $IRONIC_TEMPLATES_DIR/tftpd-xinetd.template /etc/xinetd.d/tftp
371+
sudo sed -e "s|%TFTPBOOT_DIR%|$IRONIC_TFTPBOOT_DIR|g" -i /etc/xinetd.d/tftp
372+
373+
# setup tftp file mapping to satisfy requests at the root (booting) and
374+
# /tftpboot/ sub-dir (as per deploy-ironic elements)
375+
echo "r ^([^/]) $IRONIC_TFTPBOOT_DIR/\1" >$IRONIC_TFTPBOOT_DIR/map-file
376+
echo "r ^(/tftpboot/) $IRONIC_TFTPBOOT_DIR/\2" >>$IRONIC_TFTPBOOT_DIR/map-file
377+
378+
chmod -R 0755 $IRONIC_TFTPBOOT_DIR
379+
restart_service xinetd
380+
}
381+
382+
function configure_ironic_ssh_keypair {
383+
# Generating ssh key pair for stack user
384+
if [[ ! -d $IRONIC_SSH_KEY_DIR ]]; then
385+
mkdir -p $IRONIC_SSH_KEY_DIR
386+
fi
387+
if [[ ! -d $HOME/.ssh ]]; then
388+
mkdir -p $HOME/.ssh
389+
chmod 700 $HOME/.ssh
390+
fi
391+
echo -e 'n\n' | ssh-keygen -q -t rsa -P '' -f $IRONIC_KEY_FILE
392+
cat $IRONIC_KEY_FILE.pub | tee -a $IRONIC_AUTHORIZED_KEYS_FILE
393+
}
394+
395+
function ironic_ssh_check {
396+
local KEY_FILE=$1
397+
local FLOATING_IP=$2
398+
local PORT=$3
399+
local DEFAULT_INSTANCE_USER=$4
400+
local ACTIVE_TIMEOUT=$5
401+
if ! timeout $ACTIVE_TIMEOUT sh -c "while ! ssh -p $PORT -o StrictHostKeyChecking=no -i $KEY_FILE ${DEFAULT_INSTANCE_USER}@$FLOATING_IP echo success; do sleep 1; done"; then
402+
die $LINENO "server didn't become ssh-able!"
403+
fi
404+
}
405+
406+
function configure_ironic_sshd {
407+
# Ensure sshd server accepts connections from localhost only
408+
409+
SSH_CONFIG=/etc/ssh/sshd_config
410+
HOST_PORT=$IRONIC_VM_SSH_ADDRESS:$IRONIC_VM_SSH_PORT
411+
if ! sudo grep ListenAddress $SSH_CONFIG | grep $HOST_PORT; then
412+
echo "ListenAddress $HOST_PORT" | sudo tee -a $SSH_CONFIG
413+
fi
414+
415+
SSH_SERVICE_NAME=sshd
416+
if is_ubuntu; then
417+
SSH_SERVICE_NAME=ssh
418+
fi
419+
420+
restart_service $SSH_SERVICE_NAME
421+
# to ensure ssh service is up and running
422+
sleep 3
423+
ironic_ssh_check $IRONIC_SSH_KEY_DIR/$IRONIC_SSH_KEY_FILENAME $IRONIC_VM_SSH_ADDRESS $IRONIC_VM_SSH_PORT $IRONIC_SSH_USERNAME 10
424+
425+
}
426+
427+
function configure_ironic_auxiliary {
428+
configure_ironic_dirs
429+
configure_ironic_ssh_keypair
430+
configure_ironic_sshd
431+
}
432+
433+
function prepare_baremetal_basic_ops {
434+
435+
# install diskimage-builder
436+
git_clone $BM_IMAGE_BUILD_REPO $BM_IMAGE_BUILD_DIR $BM_IMAGE_BUILD_BRANCH
437+
438+
# make sure all needed service were enabled
439+
for srv in nova glance key neutron; do
440+
if ! is_service_enabled "$srv"; then
441+
die $LINENO "$srv should be enabled for ironic tests"
442+
fi
443+
done
444+
445+
SCREEN_NAME=${SCREEN_NAME:-stack}
446+
SERVICE_DIR=${SERVICE_DIR:-${DEST}/status}
447+
448+
# stop all nova services
449+
stop_nova || true
450+
451+
# remove any nova services failure status
452+
find $SERVICE_DIR/$SCREEN_NAME -name 'n-*.failure' -exec rm -f '{}' \;
453+
454+
# start them again
455+
start_nova_api
456+
start_nova
457+
458+
TOKEN=$(keystone token-get | grep ' id ' | get_field 2)
459+
die_if_not_set $LINENO TOKEN "Keystone fail to get token"
460+
461+
echo_summary "Creating and uploading baremetal images for ironic"
462+
463+
# build and upload separate deploy kernel & ramdisk
464+
upload_baremetal_deploy $TOKEN
465+
466+
create_bridge_and_vms
467+
enroll_vms
468+
configure_tftpd
469+
}
470+
471+
function cleanup_baremetal_basic_ops {
472+
rm -f $IRONIC_VM_MACS_CSV_FILE
473+
if [ -f $IRONIC_KEY_FILE ]; then
474+
KEY=`cat $IRONIC_KEY_FILE.pub`
475+
# remove public key from authorized_keys
476+
grep -v "$KEY" $IRONIC_AUTHORIZED_KEYS_FILE > temp && mv temp $IRONIC_AUTHORIZED_KEYS_FILE
477+
chmod 0600 $IRONIC_AUTHORIZED_KEYS_FILE
478+
fi
479+
sudo rm -rf $IRONIC_DATA_DIR $IRONIC_STATE_PATH
480+
sudo su $STACK_USER -c "$IRONIC_SCRIPTS_DIR/cleanup-nodes $IRONIC_VM_COUNT $IRONIC_VM_NETWORK_BRIDGE"
481+
sudo rm -rf /etc/xinetd.d/tftp /etc/init/tftpd-hpa.override
482+
restart_service xinetd
483+
}
228484

229-
# Restore xtrace
485+
# Restore xtrace + pipefail
230486
$XTRACE
487+
$PIPEFAIL
231488

232489
# Tell emacs to use shell-script-mode
233490
## Local variables:

0 commit comments

Comments
 (0)