#!/bin/bash
set -x
set -e -o pipefail

rbdpool=replicapool
rbdimage=test-demote-sb
rbdimgsize=100G
rbdimagefeatures="--image-feature layering --image-feature exclusive-lock --image-feature object-map --image-feature fast-diff"
dimage1=dummy-image1
dimage2=dummy-image2
east=east
west=west
stime=75

# deploy rbd vstart cluster
setup_mstart() {

  rm -rf /run/$east/out/*
  rm -rf /run/$west/out/*

  echo "===setup clusters ==="
 # MON=1 OSD=1 MGR=1 MDS=0 RGW=0 ../src/mstart.sh $east --short -n --without-dashboard
 # MON=1 OSD=1 MGR=1 MDS=0 RGW=0 ../src/mstart.sh $west --short -n --without-dashboard
  MON=1 OSD=1 MGR=1 MDS=0 RGW=0 ../src/mstart.sh $east --short -n --without-dashboard -o "debug rbd = 30
  debug journaler = 30
  debug rbd_mirror = 30"
  MON=1 OSD=1 MGR=1 MDS=0 RGW=0 ../src/mstart.sh $west --short -n --without-dashboard -o "debug rbd = 30
  debug journaler = 30
  debug rbd_mirror = 30"

  echo "===rbd mirror setup for clusters ==="
  key=$(../src/mrun $east ceph auth get client.admin --format=json | jq -r .[0].key)
  mon_host_a=$(grep "mon host" run/$east/ceph.conf | awk '{print $4}')
  ceph_conf=./run/$east/ceph.conf bin/rbd-mirror --mon_host $mon_host_a --key $key --debug-rbd 5 --debug-rbd-mirror 30
  mon_host_b=$(grep "mon host" run/$west/ceph.conf | awk '{print $4}')
  key=$(../src/mrun $west ceph auth get client.admin --format=json | jq -r .[0].key)
  ceph_conf=./run/$west/ceph.conf bin/rbd-mirror --mon_host $mon_host_b --key $key --debug-rbd 5 --debug-rbd-mirror 30

  echo "===create pools=="
  ../src/mrun $east ceph osd pool create $rbdpool
  ../src/mrun $west ceph osd pool create $rbdpool

  echo "make them rbd pool...\n"
  ../src/mrun $east rbd pool init $rbdpool
  ../src/mrun $west rbd pool init $rbdpool

  echo "===connect the 2 site clusters=="
  ../src/mrun $east rbd mirror pool enable --site-name $east $rbdpool image
  token=$(../src/mrun $east rbd mirror pool peer bootstrap create $rbdpool)
  ../src/mrun $west rbd mirror pool peer bootstrap import --site-name $west $rbdpool - <<< $token

}
# $1: cluster1 $2: cluster2 $3: dummy1 $4: dummy2
setup()
{
    echo "===   Creating image $1  ==="
    ../src/mrun  $1 rbd create $rbdpool/$rbdimage --size $rbdimgsize $rbdimagefeatures
    echo
    echo "===   Enabling mirror $1  ==="
    ../src/mrun  $1 rbd mirror image enable $rbdpool/$rbdimage snapshot
    echo
    echo "===   Enabling snapshot schedule 1m $1  ==="
    ../src/mrun  $1 rbd mirror snapshot schedule add --pool $rbdpool --image $rbdimage 1m
    echo "===   Sleep 30 seconds  ==="
    sleep 30
    echo "===   Enabling snapshot schedule 1m $2  ==="
    ../src/mrun  $2 rbd mirror snapshot schedule add --pool $rbdpool --image $rbdimage 1m

    echo "===   Creating dummy image $1  ==="
    ../src/mrun  $1 rbd create $rbdpool/$3 --size 1M $rbdimagefeatures
    echo
    echo "===   Enabling mirror on dummy image $1  ==="
    ../src/mrun  $1 rbd mirror image enable $rbdpool/$3 snapshot
    echo
    echo "===   Enabling snapshot schedule 1m on dummy image $1  ==="
    ../src/mrun  $1 rbd mirror snapshot schedule add --pool $rbdpool --image $3 1m

    echo "===   Creating dummy image $2  ==="
    ../src/mrun  $2 rbd create $rbdpool/$4 --size 1M $rbdimagefeatures
    echo
    echo "===   Enabling mirror on dummy image $2  ==="
    ../src/mrun  $2 rbd mirror image enable $rbdpool/$4 snapshot
    echo
    echo "===   Enabling snapshot schedule 1m on dummy image $2  ==="
    ../src/mrun  $2 rbd mirror snapshot schedule add --pool $rbdpool --image $4 1m
}

tickleDummy()
{
    echo "===   Disabling mirror on dummy image $1  ==="
    ../src/mrun  $1 rbd mirror image disable $rbdpool/$2
    echo "===   Enabling mirror on dummy image $1  ==="
    ../src/mrun  $1 rbd mirror image enable $rbdpool/$2 snapshot
    echo "===   Sleeping for 75 seconds for resync catch up   ==="
    sleep $stime
}

display()
{
    echo `date`
    echo "===   Mirror image status $east   ==="
    ../src/mrun  $east rbd mirror image status $rbdpool/$rbdimage
    echo
    echo "===   Mirror image status $west   ==="
    ../src/mrun  $west rbd mirror image status $rbdpool/$rbdimage
    echo
    echo "===   Mirror snapshot status $east  ==="
    ../src/mrun  $east rbd mirror snapshot schedule status
    #../src/mrun  $east rbd -p $rbdpool snap ls $rbdimage --all
    echo
    echo "===   Mirror snapshot status $west   ==="
    ../src/mrun  $west rbd mirror snapshot schedule status
    #../src/mrun  $west rbd -p $rbdpool snap ls $rbdimage --all
    echo
}

#Force promote cluster1 to have 2 primaries
forcePromote()
{
    echo
    echo "===   Force promote image on $1   ==="
    ../src/mrun  $1 rbd mirror image promote $rbdpool/$rbdimage --force
    echo
}

#Demote hub
demote()
{
    echo "===   Demoting $1   ==="
    ../src/mrun  $1 rbd mirror image demote $rbdpool/$rbdimage
    echo
    echo "===   Sleeping for 75 seconds for snap schedule to catch up   ==="
    sleep $stime
}

resync()
{
    echo "===   Resync $1   ==="
    ../src/mrun  $1 rbd mirror image resync $rbdpool/$rbdimage
    echo
    echo "===   Sleeping for 75 seconds for resync catch up   ==="
    sleep $stime
}

performIO()
{
    echo "===   Perform IO on $1   ==="
    ../src/mrun  $1 rbd bench --io-type write --io-size 64K --io-threads 2 --io-total 200M --io-pattern seq $rbdpool/$rbdimage
    echo
    #echo "===   Sleeping for 75 seconds for snap schedule to catch up  ==="
    #sleep $stime
}

promote()
{
    echo "===   Promote on $1   ==="
    ../src/mrun  $1 rbd mirror image promote $rbdpool/$rbdimage
    echo
    echo "===   Sleeping for 75 seconds for snap schedule to catch up   ==="
    sleep $stime
}

cleanup()
{
    removeNetworkDelay $west
    removeNetworkDelay $east

    echo "===   Force promote $1 ==="
    ../src/mrun  $1 rbd mirror image promote $rbdpool/$rbdimage --force

    echo "===   Disable mirror $1  ==="
    ../src/mrun  $1 rbd mirror image disable $rbdpool/$rbdimage

    echo "===   Delete image on $1   ==="
    ../src/mrun  $1 rbd rm $rbdpool/$rbdimage

    echo "===   Disable dummy mirror $1  ==="
    ../src/mrun  $1 rbd mirror image disable $rbdpool/$3

    echo "===   Delete dummy image on $1   ==="
    ../src/mrun  $1 rbd rm $rbdpool/$3

    echo "===   Force promote $2 ==="
    ../src/mrun  $2 rbd mirror image promote $rbdpool/$rbdimage --force

    echo "===   Disable mirror $2  ==="
    ../src/mrun  $2 rbd mirror image disable $rbdpool/$rbdimage

    echo "===   Delete image on $2 ==="
    ../src/mrun  $2 rbd rm $rbdpool/$rbdimage

    echo "===   Disable dummy mirror $2  ==="
    ../src/mrun  $2 rbd mirror image disable $rbdpool/$4

    echo "===   Delete dummy image on $2   ==="
    ../src/mrun  $2 rbd rm $rbdpool/$4
}

addNetworkDelay()
{
    echo "===   Current eth0 status on $1   ==="
    minikube ssh --profile=$1 -- sudo tc qdisc show dev eth0
    echo "===   Adding network delay of $netdelay to eth0 on $1   ==="
    minikube ssh --profile=$1 -- sudo tc qdisc add dev eth0 root netem delay 240ms
    echo "===   Modified eth0 status on $1   ==="
    minikube ssh --profile=$1 -- sudo tc qdisc show dev eth0

    echo "===   Current eth1 status on $1   ==="
    minikube ssh --profile=$1 -- sudo tc qdisc show dev eth1
    echo "===   Adding network delay of $netdelay to eth1 on $1   ==="
    minikube ssh --profile=$1 -- sudo tc qdisc add dev eth1 root netem delay 240ms
    echo "===   Modified eth1 status on $1   ==="
    minikube ssh --profile=$1 -- sudo tc qdisc show dev eth1
}

removeNetworkDelay()
{
    echo "===   Current eth0 status on $1   ==="
    minikube ssh --profile=$1 -- sudo tc qdisc show dev eth0
    echo "===   Deleting network delay of $netdelay to eth0 on $1   ==="
    minikube ssh --profile=$1 -- sudo tc qdisc del dev eth0 root netem delay 240ms
    echo "===   Modified eth0 status on $1   ==="
    minikube ssh --profile=$1 -- sudo tc qdisc show dev eth0

    echo "===   Current eth1 status on $1   ==="
    minikube ssh --profile=$1 -- sudo tc qdisc show dev eth1
    echo "===   Deleting network delay of $netdelay to eth1 on $1   ==="
    minikube ssh --profile=$1 -- sudo tc qdisc del dev eth1 root netem delay 240ms
    echo "===   Modified eth1 status on $1   ==="
    minikube ssh --profile=$1 -- sudo tc qdisc show dev eth1
}

# create
set +e
setup_mstart
cleanup $west $east $dimage1 $dimage2
set -e -o pipefail
setup $west $east $dimage1 $dimage2
display

# failover - 1
echo "Wait for initial sync completion"
sleep 30
performIO $west
#addNetworkDelay $west
display
echo "-----------------------------------------"
echo "Ensure a snapshot sync has started on $east before keypress"
echo "Press any key to continue"
read -n 1
echo "Continuing..."
echo "-----------------------------------------"
forcePromote $east
tickleDummy $east $dimage2
display
#removeNetworkDelay $west
demote $west
display
resync $west
display
