|
#!/bin/bash
|
|
set -x
|
|
set -e -o pipefail
|
|
|
|
rbdpool=data
|
|
rbdimage=a1
|
|
rbdimgsize=100G
|
|
rbdimagefeatures="--image-feature layering --image-feature exclusive-lock --image-feature object-map --image-feature fast-diff"
|
|
#rbdimagefeatures="--image-feature layering"
|
|
site1=primary #local
|
|
site2=secondary #remote
|
|
stime=75
|
|
|
|
cluster_start() {
|
|
echo "===setup clusters ==="
|
|
mon=1 osd=1 mds=0 ../src/mstart.sh $site1 --short -n -d -o "debug rbd = 20
|
|
debug librbd = 20
|
|
debug rbd_mirror=30
|
|
rbd_default_features = 61" --without-dashboard
|
|
mon=1 osd=1 mds=0 ../src/mstart.sh $site2 --short -n -d -o "debug rbd = 20
|
|
debug librbd = 20
|
|
debug rbd_mirror=30
|
|
rbd_default_features = 61" --without-dashboard
|
|
|
|
echo "===create pools=="
|
|
../src/mrun $site1 ceph osd pool create data
|
|
../src/mrun $site2 ceph osd pool create data
|
|
|
|
echo "make them rbd pool...\n"
|
|
../src/mrun $site1 rbd pool init data
|
|
../src/mrun $site2 rbd pool init data
|
|
|
|
echo "===connect the 2 site clusters=="
|
|
../src/mrun $site1 rbd mirror pool enable --site-name $site1 data image
|
|
token=$(../src/mrun $site1 rbd mirror pool peer bootstrap create data)
|
|
../src/mrun $site2 rbd mirror pool peer bootstrap import --site-name $site2 data - <<< $token
|
|
}
|
|
|
|
|
|
# 1: cluster1 2: cluster2
|
|
setup()
|
|
{
|
|
|
|
echo "=== Creating image on site $2 ==="
|
|
../src/mrun $1 rbd create $rbdpool/$rbdimage --size $rbdimgsize $rbdimagefeatures
|
|
echo
|
|
|
|
echo "=== Enabling mirror site $1 ==="
|
|
../src/mrun $1 rbd mirror image enable $rbdpool/$rbdimage journal
|
|
echo
|
|
sleep $stime
|
|
echo "=== image on site $2 should be mirrored on site $1"
|
|
|
|
}
|
|
|
|
display()
|
|
{
|
|
echo `date`
|
|
echo "=== Mirror image status $site2 ==="
|
|
../src/mrun $site2 rbd mirror image status $rbdpool/$rbdimage
|
|
echo
|
|
echo "=== Mirror image status $site1 ==="
|
|
../src/mrun $site1 rbd mirror image status $rbdpool/$rbdimage
|
|
echo
|
|
}
|
|
|
|
#Force promote cluster1 to have 2 primaries
|
|
forcePromote()
|
|
{
|
|
echo
|
|
echo "=== Force promote image on $1 ==="
|
|
../src/mrun $1 rbd mirror image promote $rbdpool/$rbdimage --force
|
|
echo
|
|
}
|
|
|
|
#Demote hub
|
|
demote()
|
|
{
|
|
echo "=== Demoting $1 ==="
|
|
../src/mrun $1 rbd mirror image demote $rbdpool/$rbdimage
|
|
echo
|
|
}
|
|
|
|
resync()
|
|
{
|
|
echo "=== Resync $1 ==="
|
|
../src/mrun $1 rbd mirror image resync $rbdpool/$rbdimage
|
|
echo
|
|
echo "=== Sleeping for 75 seconds for resync catch up ==="
|
|
sleep $stime
|
|
}
|
|
|
|
performIO()
|
|
{
|
|
echo "=== Perform IO on $1 ==="
|
|
../src/mrun $1 rbd bench --io-type write --io-size 4K --io-threads 2 --io-total 100M --io-pattern rand $rbdpool/$rbdimage
|
|
echo
|
|
|
|
}
|
|
|
|
promote()
|
|
{
|
|
echo "=== Promote on $1 ==="
|
|
../src/mrun $1 rbd mirror image promote $rbdpool/$rbdimage
|
|
echo
|
|
|
|
}
|
|
|
|
cleanup()
|
|
{
|
|
echo "=== Force promote $1 ==="
|
|
../src/mrun $1 rbd mirror image promote $rbdpool/$rbdimage --force
|
|
|
|
echo "=== Disable mirror $1 ==="
|
|
../src/mrun $1 rbd mirror image disable $rbdpool/$rbdimage
|
|
|
|
echo "=== Delete image on $1 ==="
|
|
../src/mrun $1 rbd rm $rbdpool/$rbdimage
|
|
|
|
echo "=== Force promote $2 ==="
|
|
../src/mrun $2 rbd mirror image promote $rbdpool/$rbdimage --force
|
|
|
|
echo "=== Disable mirror $2 ==="
|
|
../src/mrun $2 rbd mirror image disable $rbdpool/$rbdimage
|
|
|
|
echo "=== Delete image on $2 ==="
|
|
../src/mrun $2 rbd rm $rbdpool/$rbdimage
|
|
|
|
echo "=== kill rbd mirror daemons ==="
|
|
sudo kill -9 $(ps aux | grep "rbd-mirror" | grep "ideepik" | awk '{print $2}')
|
|
|
|
}
|
|
|
|
# $1 start mirroring on
|
|
start_mirror() {
|
|
echo "==== start mirror $1 ===="
|
|
mon_host=$(grep "mon host" run/$1/ceph.conf | awk '{print $4}')
|
|
key=$(../src/mrun $1 ceph auth get client.admin --format=json | jq -r .[0].key)
|
|
ceph_conf=./run/$1/ceph.conf bin/rbd-mirror --mon_host $mon_host --key $key --debug-rbd 5 --debug-rbd-mirror 30
|
|
}
|
|
|
|
create_mirror_image() {
|
|
echo "=== Creating image on $1 ==="
|
|
../src/mrun $1 rbd create $rbdpool/$rbdimage --size $rbdimgsize $rbdimagefeatures
|
|
|
|
echo "=== Enabling mirror site $1 ==="
|
|
../src/mrun $1 rbd mirror image enable $rbdpool/$rbdimage journal
|
|
echo
|
|
sleep $stime
|
|
echo "=== image on site $1 should be mirrored on other site ==="
|
|
}
|
|
|
|
set +e
|
|
#cluster_start
|
|
cleanup $site1 $site2
|
|
set -e -o pipefail
|
|
start_mirror $site1
|
|
create_mirror_image $site2
|
|
sleep $stime
|
|
display
|
|
demote $site2
|
|
sleep $stime
|
|
promote $site1
|
|
sleep $stime
|
|
start_mirror $site2
|
|
|