test.txt - Ceph - Ceph

Bug #12340 » test.txt

full output of the failed test - Loïc Dachary, 07/15/2015 08:48 PM

       + PS4='${BASH_SOURCE[0]}:$LINENO: ${FUNCNAME[0]}:  '
       ../qa/workunits/ceph-helpers.sh:1188: run_tests:  export PATH=:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin
       ../qa/workunits/ceph-helpers.sh:1188: run_tests:  PATH=:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin
       ../qa/workunits/ceph-helpers.sh:1189: run_tests:  export CEPH_MON=127.0.0.1:7109
       ../qa/workunits/ceph-helpers.sh:1189: run_tests:  CEPH_MON=127.0.0.1:7109
       ../qa/workunits/ceph-helpers.sh:1190: run_tests:  export CEPH_ARGS
       .../qa/workunits/ceph-helpers.sh:1191: run_tests:  uuidgen
       ../qa/workunits/ceph-helpers.sh:1191: run_tests:  CEPH_ARGS+='--fsid=3f1b7cce-0c19-4c66-990c-9bbd1a39c039 --auth-supported=none '
       ../qa/workunits/ceph-helpers.sh:1192: run_tests:  CEPH_ARGS+='--mon-host=127.0.0.1:7109 '
       ../qa/workunits/ceph-helpers.sh:1193: run_tests:  export CEPH_CONF=/dev/null
       ../qa/workunits/ceph-helpers.sh:1193: run_tests:  CEPH_CONF=/dev/null
       .../qa/workunits/ceph-helpers.sh:1195: run_tests:  set
       .../qa/workunits/ceph-helpers.sh:1195: run_tests:  sed -n -e 's/^\(test_[0-9a-z_]*\) .*/\1/p'
       ../qa/workunits/ceph-helpers.sh:1195: run_tests:  local 'funcs=test_activate_osd
       test_erasure_code_plugin_exists
       test_expect_failure
       test_get_config
       test_get_is_making_recovery_progress
       test_get_last_scrub_stamp
       test_get_not_primary
       test_get_num_active_clean
       test_get_num_pgs
       test_get_osds
       test_get_pg
       test_get_primary
       test_is_clean
       test_kill_daemons
       test_objectstore_tool
       test_repair
       test_run_mon
       test_run_osd
       test_set_config
       test_setup
       test_teardown
       test_wait_for_clean
       test_wait_for_osd'
       ../qa/workunits/ceph-helpers.sh:1196: run_tests:  local dir=testdir/ceph-helpers
       ../qa/workunits/ceph-helpers.sh:1198: run_tests:  for func in '$funcs'
       ../qa/workunits/ceph-helpers.sh:1199: run_tests:  test_activate_osd testdir/ceph-helpers
       ../qa/workunits/ceph-helpers.sh:518: test_activate_osd:  local dir=testdir/ceph-helpers
       ../qa/workunits/ceph-helpers.sh:520: test_activate_osd:  setup testdir/ceph-helpers
       ../qa/workunits/ceph-helpers.sh:92: setup:  local dir=testdir/ceph-helpers
       ../qa/workunits/ceph-helpers.sh:93: setup:  teardown testdir/ceph-helpers
       ../qa/workunits/ceph-helpers.sh:117: teardown:  local dir=testdir/ceph-helpers
       ../qa/workunits/ceph-helpers.sh:118: teardown:  kill_daemons testdir/ceph-helpers
       .../qa/workunits/ceph-helpers.sh:192: kill_daemons:  shopt -q -o xtrace
       .../qa/workunits/ceph-helpers.sh:192: kill_daemons:  echo true
       ../qa/workunits/ceph-helpers.sh:192: kill_daemons:  local trace=true
       ../qa/workunits/ceph-helpers.sh:193: kill_daemons:  true
       ../qa/workunits/ceph-helpers.sh:193: kill_daemons:  shopt -u -o xtrace
       ../qa/workunits/ceph-helpers.sh:219: kill_daemons:  return 0
       .../qa/workunits/ceph-helpers.sh:119: teardown:  stat -f -c %T .
       ../qa/workunits/ceph-helpers.sh:119: teardown:  '[' ext2/ext3 == btrfs ']'
       ../qa/workunits/ceph-helpers.sh:122: teardown:  rm -fr testdir/ceph-helpers
       ../qa/workunits/ceph-helpers.sh:94: setup:  mkdir -p testdir/ceph-helpers
       ../qa/workunits/ceph-helpers.sh:522: test_activate_osd:  run_mon testdir/ceph-helpers a
       ../qa/workunits/ceph-helpers.sh:282: run_mon:  local dir=testdir/ceph-helpers
       ../qa/workunits/ceph-helpers.sh:283: run_mon:  shift
       ../qa/workunits/ceph-helpers.sh:284: run_mon:  local id=a
       ../qa/workunits/ceph-helpers.sh:285: run_mon:  shift
       ../qa/workunits/ceph-helpers.sh:286: run_mon:  local data=testdir/ceph-helpers/a
       ../qa/workunits/ceph-helpers.sh:289: run_mon:  ceph-mon --id a --mkfs --mon-data=testdir/ceph-helpers/a --run-dir=testdir/ceph-helpers
       ceph-mon: mon.noname-a 127.0.0.1:7109/0 is local, renaming to mon.a
       ceph-mon: set fsid to 3f1b7cce-0c19-4c66-990c-9bbd1a39c039
       ceph-mon: created monfs at testdir/ceph-helpers/a for mon.a
       ../qa/workunits/ceph-helpers.sh:296: run_mon:  ceph-mon --id a --mon-osd-full-ratio=.99 --mon-data-avail-crit=1 --paxos-propose-interval=0.1 --osd-crush-chooseleaf-type=0 --osd-pool-default-erasure-code-directory=.libs --debug-mon 20 --debug-ms 20 --debug-paxos 20 --chdir= --mon-data=testdir/ceph-helpers/a '--log-file=testdir/ceph-helpers/$name.log' '--admin-socket=testdir/ceph-helpers/$cluster-$name.asok' --mon-cluster-log-file=testdir/ceph-helpers/log --run-dir=testdir/ceph-helpers '--pid-file=testdir/ceph-helpers/$name.pid'
       ../qa/workunits/ceph-helpers.sh:314: run_mon:  cat
       .../qa/workunits/ceph-helpers.sh:314: run_mon:  get_config mon a fsid
       .../qa/workunits/ceph-helpers.sh:653: get_config:  local daemon=mon
       .../qa/workunits/ceph-helpers.sh:654: get_config:  local id=a
       .../qa/workunits/ceph-helpers.sh:655: get_config:  local config=fsid
       .../qa/workunits/ceph-helpers.sh:657: get_config:  CEPH_ARGS=
       .../qa/workunits/ceph-helpers.sh:657: get_config:  ceph --format xml daemon testdir/ceph-helpers/ceph-mon.a.asok config get fsid
       .../qa/workunits/ceph-helpers.sh:660: get_config:  xmlstarlet sel -t -m //fsid -v . -n
       .../qa/workunits/ceph-helpers.sh:314: run_mon:  get_config mon a mon_host
       .../qa/workunits/ceph-helpers.sh:653: get_config:  local daemon=mon
       .../qa/workunits/ceph-helpers.sh:654: get_config:  local id=a
       .../qa/workunits/ceph-helpers.sh:655: get_config:  local config=mon_host
       .../qa/workunits/ceph-helpers.sh:660: get_config:  xmlstarlet sel -t -m //mon_host -v . -n
       .../qa/workunits/ceph-helpers.sh:657: get_config:  CEPH_ARGS=
       .../qa/workunits/ceph-helpers.sh:657: get_config:  ceph --format xml daemon testdir/ceph-helpers/ceph-mon.a.asok config get mon_host
       .../qa/workunits/ceph-helpers.sh:319: run_mon:  get_config mon a mon_initial_members
       .../qa/workunits/ceph-helpers.sh:653: get_config:  local daemon=mon
       .../qa/workunits/ceph-helpers.sh:654: get_config:  local id=a
       .../qa/workunits/ceph-helpers.sh:655: get_config:  local config=mon_initial_members
       .../qa/workunits/ceph-helpers.sh:657: get_config:  CEPH_ARGS=
       .../qa/workunits/ceph-helpers.sh:657: get_config:  ceph --format xml daemon testdir/ceph-helpers/ceph-mon.a.asok config get mon_initial_members
       .../qa/workunits/ceph-helpers.sh:660: get_config:  xmlstarlet sel -t -m //mon_initial_members -v . -n
       ../qa/workunits/ceph-helpers.sh:319: run_mon:  test -z ''
       ../qa/workunits/ceph-helpers.sh:320: run_mon:  ceph osd pool delete rbd rbd --yes-i-really-really-mean-it
       *** DEVELOPER MODE: setting PATH, PYTHONPATH and LD_LIBRARY_PATH ***
       pool 'rbd' removed
       ../qa/workunits/ceph-helpers.sh:321: run_mon:  ceph osd pool create rbd 4
       *** DEVELOPER MODE: setting PATH, PYTHONPATH and LD_LIBRARY_PATH ***
       pool 'rbd' created
       ../qa/workunits/ceph-helpers.sh:524: test_activate_osd:  run_osd testdir/ceph-helpers 0
       ../qa/workunits/ceph-helpers.sh:397: run_osd:  local dir=testdir/ceph-helpers
       ../qa/workunits/ceph-helpers.sh:398: run_osd:  shift
       ../qa/workunits/ceph-helpers.sh:399: run_osd:  local id=0
       ../qa/workunits/ceph-helpers.sh:400: run_osd:  shift
       ../qa/workunits/ceph-helpers.sh:401: run_osd:  local osd_data=testdir/ceph-helpers/0
       ../qa/workunits/ceph-helpers.sh:403: run_osd:  local ceph_disk_args
       ../qa/workunits/ceph-helpers.sh:404: run_osd:  ceph_disk_args+=' --statedir=testdir/ceph-helpers'
       ../qa/workunits/ceph-helpers.sh:405: run_osd:  ceph_disk_args+=' --sysconfdir=testdir/ceph-helpers'
       ../qa/workunits/ceph-helpers.sh:406: run_osd:  ceph_disk_args+=' --prepend-to-path='
       ../qa/workunits/ceph-helpers.sh:408: run_osd:  mkdir -p testdir/ceph-helpers/0
       ../qa/workunits/ceph-helpers.sh:409: run_osd:  ceph-disk --statedir=testdir/ceph-helpers --sysconfdir=testdir/ceph-helpers --prepend-to-path= prepare testdir/ceph-helpers/0
 -07-15 19:58:41.930854 7f37c836c7c0 -1 warning: unable to create /var/run/ceph: (13) Permission denied
 -07-15 19:58:41.944397 7fc0874367c0 -1 warning: unable to create /var/run/ceph: (13) Permission denied
 -07-15 19:58:41.960148 7f2d771be7c0 -1 warning: unable to create /var/run/ceph: (13) Permission denied
       ../qa/workunits/ceph-helpers.sh:412: run_osd:  activate_osd testdir/ceph-helpers 0
       ../qa/workunits/ceph-helpers.sh:479: activate_osd:  local dir=testdir/ceph-helpers
       ../qa/workunits/ceph-helpers.sh:480: activate_osd:  shift
       ../qa/workunits/ceph-helpers.sh:481: activate_osd:  local id=0
       ../qa/workunits/ceph-helpers.sh:482: activate_osd:  shift
       ../qa/workunits/ceph-helpers.sh:483: activate_osd:  local osd_data=testdir/ceph-helpers/0
       ../qa/workunits/ceph-helpers.sh:485: activate_osd:  local ceph_disk_args
       ../qa/workunits/ceph-helpers.sh:486: activate_osd:  ceph_disk_args+=' --statedir=testdir/ceph-helpers'
       ../qa/workunits/ceph-helpers.sh:487: activate_osd:  ceph_disk_args+=' --sysconfdir=testdir/ceph-helpers'
       ../qa/workunits/ceph-helpers.sh:488: activate_osd:  ceph_disk_args+=' --prepend-to-path='
       ../qa/workunits/ceph-helpers.sh:490: activate_osd:  local 'ceph_args=--fsid=3f1b7cce-0c19-4c66-990c-9bbd1a39c039 --auth-supported=none --mon-host=127.0.0.1:7109 '
       ../qa/workunits/ceph-helpers.sh:491: activate_osd:  ceph_args+=' --osd-backfill-full-ratio=.99'
       ../qa/workunits/ceph-helpers.sh:492: activate_osd:  ceph_args+=' --osd-failsafe-full-ratio=.99'
       ../qa/workunits/ceph-helpers.sh:493: activate_osd:  ceph_args+=' --osd-journal-size=100'
       ../qa/workunits/ceph-helpers.sh:494: activate_osd:  ceph_args+=' --osd-data=testdir/ceph-helpers/0'
       ../qa/workunits/ceph-helpers.sh:495: activate_osd:  ceph_args+=' --chdir='
       ../qa/workunits/ceph-helpers.sh:496: activate_osd:  ceph_args+=' --osd-pool-default-erasure-code-directory=.libs'
       ../qa/workunits/ceph-helpers.sh:497: activate_osd:  ceph_args+=' --osd-class-dir=.libs'
       ../qa/workunits/ceph-helpers.sh:498: activate_osd:  ceph_args+=' --run-dir=testdir/ceph-helpers'
       ../qa/workunits/ceph-helpers.sh:499: activate_osd:  ceph_args+=' --debug-osd=20'
       ../qa/workunits/ceph-helpers.sh:500: activate_osd:  ceph_args+=' --log-file=testdir/ceph-helpers/$name.log'
       ../qa/workunits/ceph-helpers.sh:501: activate_osd:  ceph_args+=' --pid-file=testdir/ceph-helpers/$name.pid'
       ../qa/workunits/ceph-helpers.sh:502: activate_osd:  ceph_args+=' '
       ../qa/workunits/ceph-helpers.sh:503: activate_osd:  ceph_args+=
       ../qa/workunits/ceph-helpers.sh:504: activate_osd:  mkdir -p testdir/ceph-helpers/0
       ../qa/workunits/ceph-helpers.sh:505: activate_osd:  CEPH_ARGS='--fsid=3f1b7cce-0c19-4c66-990c-9bbd1a39c039 --auth-supported=none --mon-host=127.0.0.1:7109  --osd-backfill-full-ratio=.99 --osd-failsafe-full-ratio=.99 --osd-journal-size=100 --osd-data=testdir/ceph-helpers/0 --chdir= --osd-pool-default-erasure-code-directory=.libs --osd-class-dir=.libs --run-dir=testdir/ceph-helpers --debug-osd=20 --log-file=testdir/ceph-helpers/$name.log --pid-file=testdir/ceph-helpers/$name.pid  '
       ../qa/workunits/ceph-helpers.sh:505: activate_osd:  ceph-disk --statedir=testdir/ceph-helpers --sysconfdir=testdir/ceph-helpers --prepend-to-path= activate --mark-init=none testdir/ceph-helpers/0
       *** DEVELOPER MODE: setting PATH, PYTHONPATH and LD_LIBRARY_PATH ***
       *** DEVELOPER MODE: setting PATH, PYTHONPATH and LD_LIBRARY_PATH ***
       got monmap epoch 1
 -07-15 19:58:53.818201 7f3e2b2457c0 -1 journal FileJournal::_open: disabling aio for non-block journal.  Use journal_force_aio to force use of aio anyway
 -07-15 19:58:53.950997 7f3e2b2457c0 -1 journal FileJournal::_open: disabling aio for non-block journal.  Use journal_force_aio to force use of aio anyway
 -07-15 19:58:53.951525 7f3e2b2457c0 -1 filestore(testdir/ceph-helpers/0) could not find -1/23c2fcde/osd_superblock/0 in index: (2) No such file or directory
 -07-15 19:58:54.044609 7f3e2b2457c0 -1 created object store testdir/ceph-helpers/0 journal testdir/ceph-helpers/0/journal for osd.0 fsid 3f1b7cce-0c19-4c66-990c-9bbd1a39c039
 -07-15 19:58:54.044669 7f3e2b2457c0 -1 auth: error reading file: testdir/ceph-helpers/0/keyring: can't open testdir/ceph-helpers/0/keyring: (2) No such file or directory
 -07-15 19:58:54.044940 7f3e2b2457c0 -1 created new key in keyring testdir/ceph-helpers/0/keyring
       *** DEVELOPER MODE: setting PATH, PYTHONPATH and LD_LIBRARY_PATH ***
       added key for osd.0
       starting osd.0 at :/0 osd_data testdir/ceph-helpers/0 testdir/ceph-helpers/0/journal
       .../qa/workunits/ceph-helpers.sh:510: activate_osd:  cat testdir/ceph-helpers/0/whoami
       ../qa/workunits/ceph-helpers.sh:510: activate_osd:  '[' 0 = 0 ']'
       ../qa/workunits/ceph-helpers.sh:512: activate_osd:  ceph osd crush create-or-move 0 1 root=default host=localhost
       *** DEVELOPER MODE: setting PATH, PYTHONPATH and LD_LIBRARY_PATH ***
       create-or-move updating item name 'osd.0' weight 1 at location {host=localhost,root=default} to crush map
       ../qa/workunits/ceph-helpers.sh:514: activate_osd:  wait_for_osd up 0
       ../qa/workunits/ceph-helpers.sh:550: wait_for_osd:  local state=up
       ../qa/workunits/ceph-helpers.sh:551: wait_for_osd:  local id=0
       ../qa/workunits/ceph-helpers.sh:553: wait_for_osd:  status=1
       ../qa/workunits/ceph-helpers.sh:554: wait_for_osd:  (( i=0 ))
       ../qa/workunits/ceph-helpers.sh:554: wait_for_osd:  (( i < 120 ))
       ../qa/workunits/ceph-helpers.sh:555: wait_for_osd:  ceph osd dump
       ../qa/workunits/ceph-helpers.sh:555: wait_for_osd:  grep 'osd.0 up'
       *** DEVELOPER MODE: setting PATH, PYTHONPATH and LD_LIBRARY_PATH ***
       osd.0 up   in  weight 1 up_from 5 up_thru 6 down_at 0 last_clean_interval [0,0) 127.0.0.1:6812/31353 127.0.0.1:6813/31353 127.0.0.1:6814/31353 127.0.0.1:6815/31353 exists,up bcb50905-0273-4031-8ea1-50c66b629bb3
       ../qa/workunits/ceph-helpers.sh:558: wait_for_osd:  status=0
       ../qa/workunits/ceph-helpers.sh:559: wait_for_osd:  break
       ../qa/workunits/ceph-helpers.sh:562: wait_for_osd:  return 0
       .../qa/workunits/ceph-helpers.sh:526: test_activate_osd:  CEPH_ARGS=
       .../qa/workunits/ceph-helpers.sh:526: test_activate_osd:  ceph --format=json daemon testdir/ceph-helpers//ceph-osd.0.asok config get osd_max_backfills
       *** DEVELOPER MODE: setting PATH, PYTHONPATH and LD_LIBRARY_PATH ***
       ../qa/workunits/ceph-helpers.sh:526: test_activate_osd:  local 'backfills={"osd_max_backfills":"10"}'
       ../qa/workunits/ceph-helpers.sh:527: test_activate_osd:  test '{"osd_max_backfills":"10"}' = '{"osd_max_backfills":"10"}'
       ../qa/workunits/ceph-helpers.sh:529: test_activate_osd:  kill_daemons testdir/ceph-helpers TERM osd
       .../qa/workunits/ceph-helpers.sh:192: kill_daemons:  shopt -q -o xtrace
       .../qa/workunits/ceph-helpers.sh:192: kill_daemons:  echo true
       ../qa/workunits/ceph-helpers.sh:192: kill_daemons:  local trace=true
       ../qa/workunits/ceph-helpers.sh:193: kill_daemons:  true
       ../qa/workunits/ceph-helpers.sh:193: kill_daemons:  shopt -u -o xtrace
       ../qa/workunits/ceph-helpers.sh:219: kill_daemons:  return 0
       ../qa/workunits/ceph-helpers.sh:531: test_activate_osd:  activate_osd testdir/ceph-helpers 0 --osd-max-backfills 20
       ../qa/workunits/ceph-helpers.sh:479: activate_osd:  local dir=testdir/ceph-helpers
       ../qa/workunits/ceph-helpers.sh:480: activate_osd:  shift
       ../qa/workunits/ceph-helpers.sh:481: activate_osd:  local id=0
       ../qa/workunits/ceph-helpers.sh:482: activate_osd:  shift
       ../qa/workunits/ceph-helpers.sh:483: activate_osd:  local osd_data=testdir/ceph-helpers/0
       ../qa/workunits/ceph-helpers.sh:485: activate_osd:  local ceph_disk_args
       ../qa/workunits/ceph-helpers.sh:486: activate_osd:  ceph_disk_args+=' --statedir=testdir/ceph-helpers'
       ../qa/workunits/ceph-helpers.sh:487: activate_osd:  ceph_disk_args+=' --sysconfdir=testdir/ceph-helpers'
       ../qa/workunits/ceph-helpers.sh:488: activate_osd:  ceph_disk_args+=' --prepend-to-path='
       ../qa/workunits/ceph-helpers.sh:490: activate_osd:  local 'ceph_args=--fsid=3f1b7cce-0c19-4c66-990c-9bbd1a39c039 --auth-supported=none --mon-host=127.0.0.1:7109 '
       ../qa/workunits/ceph-helpers.sh:491: activate_osd:  ceph_args+=' --osd-backfill-full-ratio=.99'
       ../qa/workunits/ceph-helpers.sh:492: activate_osd:  ceph_args+=' --osd-failsafe-full-ratio=.99'
       ../qa/workunits/ceph-helpers.sh:493: activate_osd:  ceph_args+=' --osd-journal-size=100'
       ../qa/workunits/ceph-helpers.sh:494: activate_osd:  ceph_args+=' --osd-data=testdir/ceph-helpers/0'
       ../qa/workunits/ceph-helpers.sh:495: activate_osd:  ceph_args+=' --chdir='
       ../qa/workunits/ceph-helpers.sh:496: activate_osd:  ceph_args+=' --osd-pool-default-erasure-code-directory=.libs'
       ../qa/workunits/ceph-helpers.sh:497: activate_osd:  ceph_args+=' --osd-class-dir=.libs'
       ../qa/workunits/ceph-helpers.sh:498: activate_osd:  ceph_args+=' --run-dir=testdir/ceph-helpers'
       ../qa/workunits/ceph-helpers.sh:499: activate_osd:  ceph_args+=' --debug-osd=20'
       ../qa/workunits/ceph-helpers.sh:500: activate_osd:  ceph_args+=' --log-file=testdir/ceph-helpers/$name.log'
       ../qa/workunits/ceph-helpers.sh:501: activate_osd:  ceph_args+=' --pid-file=testdir/ceph-helpers/$name.pid'
       ../qa/workunits/ceph-helpers.sh:502: activate_osd:  ceph_args+=' '
       ../qa/workunits/ceph-helpers.sh:503: activate_osd:  ceph_args+='--osd-max-backfills 20'
       ../qa/workunits/ceph-helpers.sh:504: activate_osd:  mkdir -p testdir/ceph-helpers/0
       ../qa/workunits/ceph-helpers.sh:505: activate_osd:  CEPH_ARGS='--fsid=3f1b7cce-0c19-4c66-990c-9bbd1a39c039 --auth-supported=none --mon-host=127.0.0.1:7109  --osd-backfill-full-ratio=.99 --osd-failsafe-full-ratio=.99 --osd-journal-size=100 --osd-data=testdir/ceph-helpers/0 --chdir= --osd-pool-default-erasure-code-directory=.libs --osd-class-dir=.libs --run-dir=testdir/ceph-helpers --debug-osd=20 --log-file=testdir/ceph-helpers/$name.log --pid-file=testdir/ceph-helpers/$name.pid --osd-max-backfills 20 '
       ../qa/workunits/ceph-helpers.sh:505: activate_osd:  ceph-disk --statedir=testdir/ceph-helpers --sysconfdir=testdir/ceph-helpers --prepend-to-path= activate --mark-init=none testdir/ceph-helpers/0
       starting osd.0 at :/0 osd_data testdir/ceph-helpers/0 testdir/ceph-helpers/0/journal
       .../qa/workunits/ceph-helpers.sh:510: activate_osd:  cat testdir/ceph-helpers/0/whoami
       ../qa/workunits/ceph-helpers.sh:510: activate_osd:  '[' 0 = 0 ']'
       ../qa/workunits/ceph-helpers.sh:512: activate_osd:  ceph osd crush create-or-move 0 1 root=default host=localhost
       *** DEVELOPER MODE: setting PATH, PYTHONPATH and LD_LIBRARY_PATH ***
       create-or-move updated item name 'osd.0' weight 1 at location {host=localhost,root=default} to crush map
       ../qa/workunits/ceph-helpers.sh:514: activate_osd:  wait_for_osd up 0
       ../qa/workunits/ceph-helpers.sh:550: wait_for_osd:  local state=up
       ../qa/workunits/ceph-helpers.sh:551: wait_for_osd:  local id=0
       ../qa/workunits/ceph-helpers.sh:553: wait_for_osd:  status=1
       ../qa/workunits/ceph-helpers.sh:554: wait_for_osd:  (( i=0 ))
       ../qa/workunits/ceph-helpers.sh:554: wait_for_osd:  (( i < 120 ))
       ../qa/workunits/ceph-helpers.sh:555: wait_for_osd:  ceph osd dump
       ../qa/workunits/ceph-helpers.sh:555: wait_for_osd:  grep 'osd.0 up'
       *** DEVELOPER MODE: setting PATH, PYTHONPATH and LD_LIBRARY_PATH ***
       osd.0 up   in  weight 1 up_from 9 up_thru 9 down_at 8 last_clean_interval [5,7) 127.0.0.1:6852/4292 127.0.0.1:6853/4292 127.0.0.1:6854/4292 127.0.0.1:6855/4292 exists,up bcb50905-0273-4031-8ea1-50c66b629bb3
       ../qa/workunits/ceph-helpers.sh:558: wait_for_osd:  status=0
       ../qa/workunits/ceph-helpers.sh:559: wait_for_osd:  break
       ../qa/workunits/ceph-helpers.sh:562: wait_for_osd:  return 0
       .../qa/workunits/ceph-helpers.sh:533: test_activate_osd:  CEPH_ARGS=
       .../qa/workunits/ceph-helpers.sh:533: test_activate_osd:  ceph --format=json daemon testdir/ceph-helpers//ceph-osd.0.asok config get osd_max_backfills
       *** DEVELOPER MODE: setting PATH, PYTHONPATH and LD_LIBRARY_PATH ***
       ../qa/workunits/ceph-helpers.sh:533: test_activate_osd:  local 'backfills={"osd_max_backfills":"20"}'
       ../qa/workunits/ceph-helpers.sh:534: test_activate_osd:  test '{"osd_max_backfills":"20"}' = '{"osd_max_backfills":"20"}'
       ../qa/workunits/ceph-helpers.sh:536: test_activate_osd:  teardown testdir/ceph-helpers
       ../qa/workunits/ceph-helpers.sh:117: teardown:  local dir=testdir/ceph-helpers
       ../qa/workunits/ceph-helpers.sh:118: teardown:  kill_daemons testdir/ceph-helpers
       .../qa/workunits/ceph-helpers.sh:192: kill_daemons:  shopt -q -o xtrace
       .../qa/workunits/ceph-helpers.sh:192: kill_daemons:  echo true
       ../qa/workunits/ceph-helpers.sh:192: kill_daemons:  local trace=true
       ../qa/workunits/ceph-helpers.sh:193: kill_daemons:  true
       ../qa/workunits/ceph-helpers.sh:193: kill_daemons:  shopt -u -o xtrace
       ../qa/workunits/ceph-helpers.sh:219: kill_daemons:  return 0
       .../qa/workunits/ceph-helpers.sh:119: teardown:  stat -f -c %T .
       ../qa/workunits/ceph-helpers.sh:119: teardown:  '[' ext2/ext3 == btrfs ']'
       ../qa/workunits/ceph-helpers.sh:122: teardown:  rm -fr testdir/ceph-helpers
       ../qa/workunits/ceph-helpers.sh:1198: run_tests:  for func in '$funcs'
       ../qa/workunits/ceph-helpers.sh:1199: run_tests:  test_erasure_code_plugin_exists testdir/ceph-helpers
       ../qa/workunits/ceph-helpers.sh:1129: test_erasure_code_plugin_exists:  local dir=testdir/ceph-helpers
       ../qa/workunits/ceph-helpers.sh:1131: test_erasure_code_plugin_exists:  setup testdir/ceph-helpers
       ../qa/workunits/ceph-helpers.sh:92: setup:  local dir=testdir/ceph-helpers
       ../qa/workunits/ceph-helpers.sh:93: setup:  teardown testdir/ceph-helpers
       ../qa/workunits/ceph-helpers.sh:117: teardown:  local dir=testdir/ceph-helpers
       ../qa/workunits/ceph-helpers.sh:118: teardown:  kill_daemons testdir/ceph-helpers
       .../qa/workunits/ceph-helpers.sh:192: kill_daemons:  shopt -q -o xtrace
       .../qa/workunits/ceph-helpers.sh:192: kill_daemons:  echo true
       ../qa/workunits/ceph-helpers.sh:192: kill_daemons:  local trace=true
       ../qa/workunits/ceph-helpers.sh:193: kill_daemons:  true
       ../qa/workunits/ceph-helpers.sh:193: kill_daemons:  shopt -u -o xtrace
       ../qa/workunits/ceph-helpers.sh:219: kill_daemons:  return 0
       .../qa/workunits/ceph-helpers.sh:119: teardown:  stat -f -c %T .
       ../qa/workunits/ceph-helpers.sh:119: teardown:  '[' ext2/ext3 == btrfs ']'
       ../qa/workunits/ceph-helpers.sh:122: teardown:  rm -fr testdir/ceph-helpers
       ../qa/workunits/ceph-helpers.sh:94: setup:  mkdir -p testdir/ceph-helpers
       ../qa/workunits/ceph-helpers.sh:1132: test_erasure_code_plugin_exists:  run_mon testdir/ceph-helpers a
       ../qa/workunits/ceph-helpers.sh:282: run_mon:  local dir=testdir/ceph-helpers
       ../qa/workunits/ceph-helpers.sh:283: run_mon:  shift
       ../qa/workunits/ceph-helpers.sh:284: run_mon:  local id=a
       ../qa/workunits/ceph-helpers.sh:285: run_mon:  shift
       ../qa/workunits/ceph-helpers.sh:286: run_mon:  local data=testdir/ceph-helpers/a
       ../qa/workunits/ceph-helpers.sh:289: run_mon:  ceph-mon --id a --mkfs --mon-data=testdir/ceph-helpers/a --run-dir=testdir/ceph-helpers
       ceph-mon: mon.noname-a 127.0.0.1:7109/0 is local, renaming to mon.a
       ceph-mon: set fsid to 3f1b7cce-0c19-4c66-990c-9bbd1a39c039
       ceph-mon: created monfs at testdir/ceph-helpers/a for mon.a
       ../qa/workunits/ceph-helpers.sh:296: run_mon:  ceph-mon --id a --mon-osd-full-ratio=.99 --mon-data-avail-crit=1 --paxos-propose-interval=0.1 --osd-crush-chooseleaf-type=0 --osd-pool-default-erasure-code-directory=.libs --debug-mon 20 --debug-ms 20 --debug-paxos 20 --chdir= --mon-data=testdir/ceph-helpers/a '--log-file=testdir/ceph-helpers/$name.log' '--admin-socket=testdir/ceph-helpers/$cluster-$name.asok' --mon-cluster-log-file=testdir/ceph-helpers/log --run-dir=testdir/ceph-helpers '--pid-file=testdir/ceph-helpers/$name.pid'
       ../qa/workunits/ceph-helpers.sh:314: run_mon:  cat
       .../qa/workunits/ceph-helpers.sh:314: run_mon:  get_config mon a fsid
       .../qa/workunits/ceph-helpers.sh:653: get_config:  local daemon=mon
       .../qa/workunits/ceph-helpers.sh:654: get_config:  local id=a
       .../qa/workunits/ceph-helpers.sh:655: get_config:  local config=fsid
       .../qa/workunits/ceph-helpers.sh:657: get_config:  CEPH_ARGS=
       .../qa/workunits/ceph-helpers.sh:657: get_config:  ceph --format xml daemon testdir/ceph-helpers/ceph-mon.a.asok config get fsid
       .../qa/workunits/ceph-helpers.sh:660: get_config:  xmlstarlet sel -t -m //fsid -v . -n
       .../qa/workunits/ceph-helpers.sh:314: run_mon:  get_config mon a mon_host
       .../qa/workunits/ceph-helpers.sh:653: get_config:  local daemon=mon
       .../qa/workunits/ceph-helpers.sh:654: get_config:  local id=a
       .../qa/workunits/ceph-helpers.sh:655: get_config:  local config=mon_host
       .../qa/workunits/ceph-helpers.sh:657: get_config:  CEPH_ARGS=
       .../qa/workunits/ceph-helpers.sh:657: get_config:  ceph --format xml daemon testdir/ceph-helpers/ceph-mon.a.asok config get mon_host
       .../qa/workunits/ceph-helpers.sh:660: get_config:  xmlstarlet sel -t -m //mon_host -v . -n
       .../qa/workunits/ceph-helpers.sh:319: run_mon:  get_config mon a mon_initial_members
       .../qa/workunits/ceph-helpers.sh:653: get_config:  local daemon=mon
       .../qa/workunits/ceph-helpers.sh:654: get_config:  local id=a
       .../qa/workunits/ceph-helpers.sh:655: get_config:  local config=mon_initial_members
       .../qa/workunits/ceph-helpers.sh:657: get_config:  CEPH_ARGS=
       .../qa/workunits/ceph-helpers.sh:657: get_config:  ceph --format xml daemon testdir/ceph-helpers/ceph-mon.a.asok config get mon_initial_members
       .../qa/workunits/ceph-helpers.sh:660: get_config:  xmlstarlet sel -t -m //mon_initial_members -v . -n
       ../qa/workunits/ceph-helpers.sh:319: run_mon:  test -z ''
       ../qa/workunits/ceph-helpers.sh:320: run_mon:  ceph osd pool delete rbd rbd --yes-i-really-really-mean-it
       *** DEVELOPER MODE: setting PATH, PYTHONPATH and LD_LIBRARY_PATH ***
       pool 'rbd' removed
       ../qa/workunits/ceph-helpers.sh:321: run_mon:  ceph osd pool create rbd 4
       *** DEVELOPER MODE: setting PATH, PYTHONPATH and LD_LIBRARY_PATH ***
       pool 'rbd' created
       ../qa/workunits/ceph-helpers.sh:1133: test_erasure_code_plugin_exists:  erasure_code_plugin_exists jerasure
       ../qa/workunits/ceph-helpers.sh:1115: erasure_code_plugin_exists:  local plugin=jerasure
       ../qa/workunits/ceph-helpers.sh:1117: erasure_code_plugin_exists:  local status
       ../qa/workunits/ceph-helpers.sh:1118: erasure_code_plugin_exists:  ceph osd erasure-code-profile set TESTPROFILE plugin=jerasure
       ../qa/workunits/ceph-helpers.sh:1119: erasure_code_plugin_exists:  grep 'jerasure.*No such file'
       ../qa/workunits/ceph-helpers.sh:1122: erasure_code_plugin_exists:  status=0
       ../qa/workunits/ceph-helpers.sh:1123: erasure_code_plugin_exists:  ceph osd erasure-code-profile rm TESTPROFILE
       *** DEVELOPER MODE: setting PATH, PYTHONPATH and LD_LIBRARY_PATH ***
       ../qa/workunits/ceph-helpers.sh:1125: erasure_code_plugin_exists:  return 0
       ../qa/workunits/ceph-helpers.sh:1134: test_erasure_code_plugin_exists:  erasure_code_plugin_exists FAKE
       ../qa/workunits/ceph-helpers.sh:1115: erasure_code_plugin_exists:  local plugin=FAKE
       ../qa/workunits/ceph-helpers.sh:1117: erasure_code_plugin_exists:  local status
       ../qa/workunits/ceph-helpers.sh:1118: erasure_code_plugin_exists:  ceph osd erasure-code-profile set TESTPROFILE plugin=FAKE
       ../qa/workunits/ceph-helpers.sh:1119: erasure_code_plugin_exists:  grep 'FAKE.*No such file'
       Error EIO: load dlopen(.libs/libec_FAKE.so): .libs/libec_FAKE.so: cannot open shared object file: No such file or directory
       ../qa/workunits/ceph-helpers.sh:1120: erasure_code_plugin_exists:  status=1
       ../qa/workunits/ceph-helpers.sh:1125: erasure_code_plugin_exists:  return 1
       ../qa/workunits/ceph-helpers.sh:1135: test_erasure_code_plugin_exists:  teardown testdir/ceph-helpers
       ../qa/workunits/ceph-helpers.sh:117: teardown:  local dir=testdir/ceph-helpers
       ../qa/workunits/ceph-helpers.sh:118: teardown:  kill_daemons testdir/ceph-helpers
       .../qa/workunits/ceph-helpers.sh:192: kill_daemons:  shopt -q -o xtrace
       .../qa/workunits/ceph-helpers.sh:192: kill_daemons:  echo true
       ../qa/workunits/ceph-helpers.sh:192: kill_daemons:  local trace=true
       ../qa/workunits/ceph-helpers.sh:193: kill_daemons:  true
       ../qa/workunits/ceph-helpers.sh:193: kill_daemons:  shopt -u -o xtrace
       ../qa/workunits/ceph-helpers.sh:219: kill_daemons:  return 0
       .../qa/workunits/ceph-helpers.sh:119: teardown:  stat -f -c %T .
       ../qa/workunits/ceph-helpers.sh:119: teardown:  '[' ext2/ext3 == btrfs ']'
       ../qa/workunits/ceph-helpers.sh:122: teardown:  rm -fr testdir/ceph-helpers
       ../qa/workunits/ceph-helpers.sh:1198: run_tests:  for func in '$funcs'
       ../qa/workunits/ceph-helpers.sh:1199: run_tests:  test_expect_failure testdir/ceph-helpers
       ../qa/workunits/ceph-helpers.sh:1092: test_expect_failure:  local dir=testdir/ceph-helpers
       ../qa/workunits/ceph-helpers.sh:1094: test_expect_failure:  setup testdir/ceph-helpers
       ../qa/workunits/ceph-helpers.sh:92: setup:  local dir=testdir/ceph-helpers
       ../qa/workunits/ceph-helpers.sh:93: setup:  teardown testdir/ceph-helpers
       ../qa/workunits/ceph-helpers.sh:117: teardown:  local dir=testdir/ceph-helpers
       ../qa/workunits/ceph-helpers.sh:118: teardown:  kill_daemons testdir/ceph-helpers
       .../qa/workunits/ceph-helpers.sh:192: kill_daemons:  shopt -q -o xtrace
       .../qa/workunits/ceph-helpers.sh:192: kill_daemons:  echo true
       ../qa/workunits/ceph-helpers.sh:192: kill_daemons:  local trace=true
       ../qa/workunits/ceph-helpers.sh:193: kill_daemons:  true
       ../qa/workunits/ceph-helpers.sh:193: kill_daemons:  shopt -u -o xtrace
       ../qa/workunits/ceph-helpers.sh:219: kill_daemons:  return 0
       .../qa/workunits/ceph-helpers.sh:119: teardown:  stat -f -c %T .
       ../qa/workunits/ceph-helpers.sh:119: teardown:  '[' ext2/ext3 == btrfs ']'
       ../qa/workunits/ceph-helpers.sh:122: teardown:  rm -fr testdir/ceph-helpers
       ../qa/workunits/ceph-helpers.sh:94: setup:  mkdir -p testdir/ceph-helpers
       ../qa/workunits/ceph-helpers.sh:1095: test_expect_failure:  expect_failure testdir/ceph-helpers FAIL bash -c 'echo FAIL ; exit 1'
       ../qa/workunits/ceph-helpers.sh:1071: expect_failure:  local dir=testdir/ceph-helpers
       ../qa/workunits/ceph-helpers.sh:1072: expect_failure:  shift
       ../qa/workunits/ceph-helpers.sh:1073: expect_failure:  local expected=FAIL
       ../qa/workunits/ceph-helpers.sh:1074: expect_failure:  shift
       ../qa/workunits/ceph-helpers.sh:1075: expect_failure:  local success
       ../qa/workunits/ceph-helpers.sh:1077: expect_failure:  bash -c 'echo FAIL ; exit 1'
       ../qa/workunits/ceph-helpers.sh:1080: expect_failure:  success=false
       ../qa/workunits/ceph-helpers.sh:1083: expect_failure:  false
       ../qa/workunits/ceph-helpers.sh:1083: expect_failure:  grep --quiet FAIL testdir/ceph-helpers/out
       ../qa/workunits/ceph-helpers.sh:1087: expect_failure:  return 0
       ../qa/workunits/ceph-helpers.sh:1097: test_expect_failure:  expect_failure testdir/ceph-helpers FAIL bash -c 'echo FAIL ; exit 0'
       ../qa/workunits/ceph-helpers.sh:1071: expect_failure:  local dir=testdir/ceph-helpers
       ../qa/workunits/ceph-helpers.sh:1072: expect_failure:  shift
       ../qa/workunits/ceph-helpers.sh:1073: expect_failure:  local expected=FAIL
       ../qa/workunits/ceph-helpers.sh:1074: expect_failure:  shift
       ../qa/workunits/ceph-helpers.sh:1075: expect_failure:  local success
       ../qa/workunits/ceph-helpers.sh:1077: expect_failure:  bash -c 'echo FAIL ; exit 0'
       ../qa/workunits/ceph-helpers.sh:1078: expect_failure:  success=true
       ../qa/workunits/ceph-helpers.sh:1083: expect_failure:  true
       ../qa/workunits/ceph-helpers.sh:1084: expect_failure:  cat testdir/ceph-helpers/out
       FAIL
       ../qa/workunits/ceph-helpers.sh:1085: expect_failure:  return 1
       ../qa/workunits/ceph-helpers.sh:1098: test_expect_failure:  grep --quiet FAIL testdir/ceph-helpers/out
       ../qa/workunits/ceph-helpers.sh:1100: test_expect_failure:  expect_failure testdir/ceph-helpers FAIL bash -c 'echo UNEXPECTED ; exit 1'
       ../qa/workunits/ceph-helpers.sh:1071: expect_failure:  local dir=testdir/ceph-helpers
       ../qa/workunits/ceph-helpers.sh:1072: expect_failure:  shift
       ../qa/workunits/ceph-helpers.sh:1073: expect_failure:  local expected=FAIL
       ../qa/workunits/ceph-helpers.sh:1074: expect_failure:  shift
       ../qa/workunits/ceph-helpers.sh:1075: expect_failure:  local success
       ../qa/workunits/ceph-helpers.sh:1077: expect_failure:  bash -c 'echo UNEXPECTED ; exit 1'
       ../qa/workunits/ceph-helpers.sh:1080: expect_failure:  success=false
       ../qa/workunits/ceph-helpers.sh:1083: expect_failure:  false
       ../qa/workunits/ceph-helpers.sh:1083: expect_failure:  grep --quiet FAIL testdir/ceph-helpers/out
       ../qa/workunits/ceph-helpers.sh:1084: expect_failure:  cat testdir/ceph-helpers/out
       UNEXPECTED
       ../qa/workunits/ceph-helpers.sh:1085: expect_failure:  return 1
       ../qa/workunits/ceph-helpers.sh:1101: test_expect_failure:  grep --quiet FAIL testdir/ceph-helpers/out
       ../qa/workunits/ceph-helpers.sh:1102: test_expect_failure:  teardown testdir/ceph-helpers
       ../qa/workunits/ceph-helpers.sh:117: teardown:  local dir=testdir/ceph-helpers
       ../qa/workunits/ceph-helpers.sh:118: teardown:  kill_daemons testdir/ceph-helpers
       .../qa/workunits/ceph-helpers.sh:192: kill_daemons:  shopt -q -o xtrace
       .../qa/workunits/ceph-helpers.sh:192: kill_daemons:  echo true
       ../qa/workunits/ceph-helpers.sh:192: kill_daemons:  local trace=true
       ../qa/workunits/ceph-helpers.sh:193: kill_daemons:  true
       ../qa/workunits/ceph-helpers.sh:193: kill_daemons:  shopt -u -o xtrace
       ../qa/workunits/ceph-helpers.sh:219: kill_daemons:  return 0
       .../qa/workunits/ceph-helpers.sh:119: teardown:  stat -f -c %T .
       ../qa/workunits/ceph-helpers.sh:119: teardown:  '[' ext2/ext3 == btrfs ']'
       ../qa/workunits/ceph-helpers.sh:122: teardown:  rm -fr testdir/ceph-helpers
       ../qa/workunits/ceph-helpers.sh:1198: run_tests:  for func in '$funcs'
       ../qa/workunits/ceph-helpers.sh:1199: run_tests:  test_get_config testdir/ceph-helpers
       ../qa/workunits/ceph-helpers.sh:664: test_get_config:  local dir=testdir/ceph-helpers
       ../qa/workunits/ceph-helpers.sh:667: test_get_config:  setup testdir/ceph-helpers
       ../qa/workunits/ceph-helpers.sh:92: setup:  local dir=testdir/ceph-helpers
       ../qa/workunits/ceph-helpers.sh:93: setup:  teardown testdir/ceph-helpers
       ../qa/workunits/ceph-helpers.sh:117: teardown:  local dir=testdir/ceph-helpers
       ../qa/workunits/ceph-helpers.sh:118: teardown:  kill_daemons testdir/ceph-helpers
       .../qa/workunits/ceph-helpers.sh:192: kill_daemons:  shopt -q -o xtrace
       .../qa/workunits/ceph-helpers.sh:192: kill_daemons:  echo true
       ../qa/workunits/ceph-helpers.sh:192: kill_daemons:  local trace=true
       ../qa/workunits/ceph-helpers.sh:193: kill_daemons:  true
       ../qa/workunits/ceph-helpers.sh:193: kill_daemons:  shopt -u -o xtrace
       ../qa/workunits/ceph-helpers.sh:219: kill_daemons:  return 0
       .../qa/workunits/ceph-helpers.sh:119: teardown:  stat -f -c %T .
       ../qa/workunits/ceph-helpers.sh:119: teardown:  '[' ext2/ext3 == btrfs ']'
       ../qa/workunits/ceph-helpers.sh:122: teardown:  rm -fr testdir/ceph-helpers
       ../qa/workunits/ceph-helpers.sh:94: setup:  mkdir -p testdir/ceph-helpers
       ../qa/workunits/ceph-helpers.sh:668: test_get_config:  run_mon testdir/ceph-helpers a --osd_pool_default_size=1
       ../qa/workunits/ceph-helpers.sh:282: run_mon:  local dir=testdir/ceph-helpers
       ../qa/workunits/ceph-helpers.sh:283: run_mon:  shift
       ../qa/workunits/ceph-helpers.sh:284: run_mon:  local id=a
       ../qa/workunits/ceph-helpers.sh:285: run_mon:  shift
       ../qa/workunits/ceph-helpers.sh:286: run_mon:  local data=testdir/ceph-helpers/a
       ../qa/workunits/ceph-helpers.sh:289: run_mon:  ceph-mon --id a --mkfs --mon-data=testdir/ceph-helpers/a --run-dir=testdir/ceph-helpers --osd_pool_default_size=1
       ceph-mon: mon.noname-a 127.0.0.1:7109/0 is local, renaming to mon.a
       ceph-mon: set fsid to 3f1b7cce-0c19-4c66-990c-9bbd1a39c039
       ceph-mon: created monfs at testdir/ceph-helpers/a for mon.a
       ../qa/workunits/ceph-helpers.sh:296: run_mon:  ceph-mon --id a --mon-osd-full-ratio=.99 --mon-data-avail-crit=1 --paxos-propose-interval=0.1 --osd-crush-chooseleaf-type=0 --osd-pool-default-erasure-code-directory=.libs --debug-mon 20 --debug-ms 20 --debug-paxos 20 --chdir= --mon-data=testdir/ceph-helpers/a '--log-file=testdir/ceph-helpers/$name.log' '--admin-socket=testdir/ceph-helpers/$cluster-$name.asok' --mon-cluster-log-file=testdir/ceph-helpers/log --run-dir=testdir/ceph-helpers '--pid-file=testdir/ceph-helpers/$name.pid' --osd_pool_default_size=1
       ../qa/workunits/ceph-helpers.sh:314: run_mon:  cat
       .../qa/workunits/ceph-helpers.sh:314: run_mon:  get_config mon a fsid
       .../qa/workunits/ceph-helpers.sh:653: get_config:  local daemon=mon
       .../qa/workunits/ceph-helpers.sh:654: get_config:  local id=a
       .../qa/workunits/ceph-helpers.sh:655: get_config:  local config=fsid
       .../qa/workunits/ceph-helpers.sh:657: get_config:  CEPH_ARGS=
       .../qa/workunits/ceph-helpers.sh:657: get_config:  ceph --format xml daemon testdir/ceph-helpers/ceph-mon.a.asok config get fsid
       .../qa/workunits/ceph-helpers.sh:660: get_config:  xmlstarlet sel -t -m //fsid -v . -n
       .../qa/workunits/ceph-helpers.sh:314: run_mon:  get_config mon a mon_host
       .../qa/workunits/ceph-helpers.sh:653: get_config:  local daemon=mon
       .../qa/workunits/ceph-helpers.sh:654: get_config:  local id=a
       .../qa/workunits/ceph-helpers.sh:655: get_config:  local config=mon_host
       .../qa/workunits/ceph-helpers.sh:657: get_config:  CEPH_ARGS=
       .../qa/workunits/ceph-helpers.sh:657: get_config:  ceph --format xml daemon testdir/ceph-helpers/ceph-mon.a.asok config get mon_host
       .../qa/workunits/ceph-helpers.sh:660: get_config:  xmlstarlet sel -t -m //mon_host -v . -n
       .../qa/workunits/ceph-helpers.sh:319: run_mon:  get_config mon a mon_initial_members
       .../qa/workunits/ceph-helpers.sh:653: get_config:  local daemon=mon
       .../qa/workunits/ceph-helpers.sh:654: get_config:  local id=a
       .../qa/workunits/ceph-helpers.sh:655: get_config:  local config=mon_initial_members
       .../qa/workunits/ceph-helpers.sh:657: get_config:  CEPH_ARGS=
       .../qa/workunits/ceph-helpers.sh:657: get_config:  ceph --format xml daemon testdir/ceph-helpers/ceph-mon.a.asok config get mon_initial_members
       .../qa/workunits/ceph-helpers.sh:660: get_config:  xmlstarlet sel -t -m //mon_initial_members -v . -n
       ../qa/workunits/ceph-helpers.sh:319: run_mon:  test -z ''
       ../qa/workunits/ceph-helpers.sh:320: run_mon:  ceph osd pool delete rbd rbd --yes-i-really-really-mean-it
       *** DEVELOPER MODE: setting PATH, PYTHONPATH and LD_LIBRARY_PATH ***
       pool 'rbd' removed
       ../qa/workunits/ceph-helpers.sh:321: run_mon:  ceph osd pool create rbd 4
       *** DEVELOPER MODE: setting PATH, PYTHONPATH and LD_LIBRARY_PATH ***
       pool 'rbd' created
       .../qa/workunits/ceph-helpers.sh:669: test_get_config:  get_config mon a osd_pool_default_size
       .../qa/workunits/ceph-helpers.sh:653: get_config:  local daemon=mon
       .../qa/workunits/ceph-helpers.sh:654: get_config:  local id=a
       .../qa/workunits/ceph-helpers.sh:655: get_config:  local config=osd_pool_default_size
       .../qa/workunits/ceph-helpers.sh:660: get_config:  xmlstarlet sel -t -m //osd_pool_default_size -v . -n
       .../qa/workunits/ceph-helpers.sh:657: get_config:  CEPH_ARGS=
       .../qa/workunits/ceph-helpers.sh:657: get_config:  ceph --format xml daemon testdir/ceph-helpers/ceph-mon.a.asok config get osd_pool_default_size
       ../qa/workunits/ceph-helpers.sh:669: test_get_config:  test 1 = 1
       ../qa/workunits/ceph-helpers.sh:670: test_get_config:  run_osd testdir/ceph-helpers 0 --osd_max_scrubs=3
       ../qa/workunits/ceph-helpers.sh:397: run_osd:  local dir=testdir/ceph-helpers
       ../qa/workunits/ceph-helpers.sh:398: run_osd:  shift
       ../qa/workunits/ceph-helpers.sh:399: run_osd:  local id=0
       ../qa/workunits/ceph-helpers.sh:400: run_osd:  shift
       ../qa/workunits/ceph-helpers.sh:401: run_osd:  local osd_data=testdir/ceph-helpers/0
       ../qa/workunits/ceph-helpers.sh:403: run_osd:  local ceph_disk_args
       ../qa/workunits/ceph-helpers.sh:404: run_osd:  ceph_disk_args+=' --statedir=testdir/ceph-helpers'
       ../qa/workunits/ceph-helpers.sh:405: run_osd:  ceph_disk_args+=' --sysconfdir=testdir/ceph-helpers'
       ../qa/workunits/ceph-helpers.sh:406: run_osd:  ceph_disk_args+=' --prepend-to-path='
       ../qa/workunits/ceph-helpers.sh:408: run_osd:  mkdir -p testdir/ceph-helpers/0
       ../qa/workunits/ceph-helpers.sh:409: run_osd:  ceph-disk --statedir=testdir/ceph-helpers --sysconfdir=testdir/ceph-helpers --prepend-to-path= prepare testdir/ceph-helpers/0
 -07-15 19:59:08.338828 7fadc81c37c0 -1 warning: unable to create /var/run/ceph: (13) Permission denied
 -07-15 19:59:08.352391 7f6d0f9687c0 -1 warning: unable to create /var/run/ceph: (13) Permission denied
 -07-15 19:59:08.366205 7fd04b0b27c0 -1 warning: unable to create /var/run/ceph: (13) Permission denied
       ../qa/workunits/ceph-helpers.sh:412: run_osd:  activate_osd testdir/ceph-helpers 0 --osd_max_scrubs=3
       ../qa/workunits/ceph-helpers.sh:479: activate_osd:  local dir=testdir/ceph-helpers
       ../qa/workunits/ceph-helpers.sh:480: activate_osd:  shift
       ../qa/workunits/ceph-helpers.sh:481: activate_osd:  local id=0
       ../qa/workunits/ceph-helpers.sh:482: activate_osd:  shift
       ../qa/workunits/ceph-helpers.sh:483: activate_osd:  local osd_data=testdir/ceph-helpers/0
       ../qa/workunits/ceph-helpers.sh:485: activate_osd:  local ceph_disk_args
       ../qa/workunits/ceph-helpers.sh:486: activate_osd:  ceph_disk_args+=' --statedir=testdir/ceph-helpers'
       ../qa/workunits/ceph-helpers.sh:487: activate_osd:  ceph_disk_args+=' --sysconfdir=testdir/ceph-helpers'
       ../qa/workunits/ceph-helpers.sh:488: activate_osd:  ceph_disk_args+=' --prepend-to-path='
       ../qa/workunits/ceph-helpers.sh:490: activate_osd:  local 'ceph_args=--fsid=3f1b7cce-0c19-4c66-990c-9bbd1a39c039 --auth-supported=none --mon-host=127.0.0.1:7109 '
       ../qa/workunits/ceph-helpers.sh:491: activate_osd:  ceph_args+=' --osd-backfill-full-ratio=.99'
       ../qa/workunits/ceph-helpers.sh:492: activate_osd:  ceph_args+=' --osd-failsafe-full-ratio=.99'
       ../qa/workunits/ceph-helpers.sh:493: activate_osd:  ceph_args+=' --osd-journal-size=100'
       ../qa/workunits/ceph-helpers.sh:494: activate_osd:  ceph_args+=' --osd-data=testdir/ceph-helpers/0'
       ../qa/workunits/ceph-helpers.sh:495: activate_osd:  ceph_args+=' --chdir='
       ../qa/workunits/ceph-helpers.sh:496: activate_osd:  ceph_args+=' --osd-pool-default-erasure-code-directory=.libs'
       ../qa/workunits/ceph-helpers.sh:497: activate_osd:  ceph_args+=' --osd-class-dir=.libs'
       ../qa/workunits/ceph-helpers.sh:498: activate_osd:  ceph_args+=' --run-dir=testdir/ceph-helpers'
       ../qa/workunits/ceph-helpers.sh:499: activate_osd:  ceph_args+=' --debug-osd=20'
       ../qa/workunits/ceph-helpers.sh:500: activate_osd:  ceph_args+=' --log-file=testdir/ceph-helpers/$name.log'
       ../qa/workunits/ceph-helpers.sh:501: activate_osd:  ceph_args+=' --pid-file=testdir/ceph-helpers/$name.pid'
       ../qa/workunits/ceph-helpers.sh:502: activate_osd:  ceph_args+=' '
       ../qa/workunits/ceph-helpers.sh:503: activate_osd:  ceph_args+=--osd_max_scrubs=3
       ../qa/workunits/ceph-helpers.sh:504: activate_osd:  mkdir -p testdir/ceph-helpers/0
       ../qa/workunits/ceph-helpers.sh:505: activate_osd:  CEPH_ARGS='--fsid=3f1b7cce-0c19-4c66-990c-9bbd1a39c039 --auth-supported=none --mon-host=127.0.0.1:7109  --osd-backfill-full-ratio=.99 --osd-failsafe-full-ratio=.99 --osd-journal-size=100 --osd-data=testdir/ceph-helpers/0 --chdir= --osd-pool-default-erasure-code-directory=.libs --osd-class-dir=.libs --run-dir=testdir/ceph-helpers --debug-osd=20 --log-file=testdir/ceph-helpers/$name.log --pid-file=testdir/ceph-helpers/$name.pid --osd_max_scrubs=3 '
       ../qa/workunits/ceph-helpers.sh:505: activate_osd:  ceph-disk --statedir=testdir/ceph-helpers --sysconfdir=testdir/ceph-helpers --prepend-to-path= activate --mark-init=none testdir/ceph-helpers/0
       *** DEVELOPER MODE: setting PATH, PYTHONPATH and LD_LIBRARY_PATH ***
       *** DEVELOPER MODE: setting PATH, PYTHONPATH and LD_LIBRARY_PATH ***
       got monmap epoch 1
 -07-15 19:59:10.796843 7f9b078ae7c0 -1 journal FileJournal::_open: disabling aio for non-block journal.  Use journal_force_aio to force use of aio anyway
 -07-15 19:59:10.853431 7f9b078ae7c0 -1 journal FileJournal::_open: disabling aio for non-block journal.  Use journal_force_aio to force use of aio anyway
 -07-15 19:59:10.853919 7f9b078ae7c0 -1 filestore(testdir/ceph-helpers/0) could not find -1/23c2fcde/osd_superblock/0 in index: (2) No such file or directory
 -07-15 19:59:10.903248 7f9b078ae7c0 -1 created object store testdir/ceph-helpers/0 journal testdir/ceph-helpers/0/journal for osd.0 fsid 3f1b7cce-0c19-4c66-990c-9bbd1a39c039
 -07-15 19:59:10.903367 7f9b078ae7c0 -1 auth: error reading file: testdir/ceph-helpers/0/keyring: can't open testdir/ceph-helpers/0/keyring: (2) No such file or directory
 -07-15 19:59:10.903553 7f9b078ae7c0 -1 created new key in keyring testdir/ceph-helpers/0/keyring
       *** DEVELOPER MODE: setting PATH, PYTHONPATH and LD_LIBRARY_PATH ***
       added key for osd.0
       starting osd.0 at :/0 osd_data testdir/ceph-helpers/0 testdir/ceph-helpers/0/journal
       .../qa/workunits/ceph-helpers.sh:510: activate_osd:  cat testdir/ceph-helpers/0/whoami
       ../qa/workunits/ceph-helpers.sh:510: activate_osd:  '[' 0 = 0 ']'
       ../qa/workunits/ceph-helpers.sh:512: activate_osd:  ceph osd crush create-or-move 0 1 root=default host=localhost
       *** DEVELOPER MODE: setting PATH, PYTHONPATH and LD_LIBRARY_PATH ***
       create-or-move updating item name 'osd.0' weight 1 at location {host=localhost,root=default} to crush map
       ../qa/workunits/ceph-helpers.sh:514: activate_osd:  wait_for_osd up 0
       ../qa/workunits/ceph-helpers.sh:550: wait_for_osd:  local state=up
       ../qa/workunits/ceph-helpers.sh:551: wait_for_osd:  local id=0
       ../qa/workunits/ceph-helpers.sh:553: wait_for_osd:  status=1
       ../qa/workunits/ceph-helpers.sh:554: wait_for_osd:  (( i=0 ))
       ../qa/workunits/ceph-helpers.sh:554: wait_for_osd:  (( i < 120 ))
       ../qa/workunits/ceph-helpers.sh:555: wait_for_osd:  ceph osd dump
       ../qa/workunits/ceph-helpers.sh:555: wait_for_osd:  grep 'osd.0 up'
       *** DEVELOPER MODE: setting PATH, PYTHONPATH and LD_LIBRARY_PATH ***
       osd.0 up   in  weight 1 up_from 5 up_thru 6 down_at 0 last_clean_interval [0,0) 127.0.0.1:6878/18192 127.0.0.1:6879/18192 127.0.0.1:6880/18192 127.0.0.1:6881/18192 exists,up dfe1c114-a792-49d9-9c8f-82bbc71a263e
       ../qa/workunits/ceph-helpers.sh:558: wait_for_osd:  status=0
       ../qa/workunits/ceph-helpers.sh:559: wait_for_osd:  break
       ../qa/workunits/ceph-helpers.sh:562: wait_for_osd:  return 0
       .../qa/workunits/ceph-helpers.sh:671: test_get_config:  get_config osd 0 osd_max_scrubs
       .../qa/workunits/ceph-helpers.sh:653: get_config:  local daemon=osd
       .../qa/workunits/ceph-helpers.sh:654: get_config:  local id=0
       .../qa/workunits/ceph-helpers.sh:655: get_config:  local config=osd_max_scrubs
       .../qa/workunits/ceph-helpers.sh:657: get_config:  CEPH_ARGS=
       .../qa/workunits/ceph-helpers.sh:657: get_config:  ceph --format xml daemon testdir/ceph-helpers/ceph-osd.0.asok config get osd_max_scrubs
       .../qa/workunits/ceph-helpers.sh:660: get_config:  xmlstarlet sel -t -m //osd_max_scrubs -v . -n
       ../qa/workunits/ceph-helpers.sh:671: test_get_config:  test 3 = 3
       ../qa/workunits/ceph-helpers.sh:672: test_get_config:  teardown testdir/ceph-helpers
       ../qa/workunits/ceph-helpers.sh:117: teardown:  local dir=testdir/ceph-helpers
       ../qa/workunits/ceph-helpers.sh:118: teardown:  kill_daemons testdir/ceph-helpers
       .../qa/workunits/ceph-helpers.sh:192: kill_daemons:  shopt -q -o xtrace
       .../qa/workunits/ceph-helpers.sh:192: kill_daemons:  echo true
       ../qa/workunits/ceph-helpers.sh:192: kill_daemons:  local trace=true
       ../qa/workunits/ceph-helpers.sh:193: kill_daemons:  true
       ../qa/workunits/ceph-helpers.sh:193: kill_daemons:  shopt -u -o xtrace
       ../qa/workunits/ceph-helpers.sh:219: kill_daemons:  return 0
       .../qa/workunits/ceph-helpers.sh:119: teardown:  stat -f -c %T .
       ../qa/workunits/ceph-helpers.sh:119: teardown:  '[' ext2/ext3 == btrfs ']'
       ../qa/workunits/ceph-helpers.sh:122: teardown:  rm -fr testdir/ceph-helpers
       ../qa/workunits/ceph-helpers.sh:1198: run_tests:  for func in '$funcs'
       ../qa/workunits/ceph-helpers.sh:1199: run_tests:  test_get_is_making_recovery_progress testdir/ceph-helpers
       ../qa/workunits/ceph-helpers.sh:844: test_get_is_making_recovery_progress:  local dir=testdir/ceph-helpers
       ../qa/workunits/ceph-helpers.sh:846: test_get_is_making_recovery_progress:  setup testdir/ceph-helpers
       ../qa/workunits/ceph-helpers.sh:92: setup:  local dir=testdir/ceph-helpers
       ../qa/workunits/ceph-helpers.sh:93: setup:  teardown testdir/ceph-helpers
       ../qa/workunits/ceph-helpers.sh:117: teardown:  local dir=testdir/ceph-helpers
       ../qa/workunits/ceph-helpers.sh:118: teardown:  kill_daemons testdir/ceph-helpers
       .../qa/workunits/ceph-helpers.sh:192: kill_daemons:  shopt -q -o xtrace
       .../qa/workunits/ceph-helpers.sh:192: kill_daemons:  echo true
       ../qa/workunits/ceph-helpers.sh:192: kill_daemons:  local trace=true
       ../qa/workunits/ceph-helpers.sh:193: kill_daemons:  true
       ../qa/workunits/ceph-helpers.sh:193: kill_daemons:  shopt -u -o xtrace
       ../qa/workunits/ceph-helpers.sh:219: kill_daemons:  return 0
       .../qa/workunits/ceph-helpers.sh:119: teardown:  stat -f -c %T .
       ../qa/workunits/ceph-helpers.sh:119: teardown:  '[' ext2/ext3 == btrfs ']'
       ../qa/workunits/ceph-helpers.sh:122: teardown:  rm -fr testdir/ceph-helpers
       ../qa/workunits/ceph-helpers.sh:94: setup:  mkdir -p testdir/ceph-helpers
       ../qa/workunits/ceph-helpers.sh:847: test_get_is_making_recovery_progress:  run_mon testdir/ceph-helpers a
       ../qa/workunits/ceph-helpers.sh:282: run_mon:  local dir=testdir/ceph-helpers
       ../qa/workunits/ceph-helpers.sh:283: run_mon:  shift
       ../qa/workunits/ceph-helpers.sh:284: run_mon:  local id=a
       ../qa/workunits/ceph-helpers.sh:285: run_mon:  shift
       ../qa/workunits/ceph-helpers.sh:286: run_mon:  local data=testdir/ceph-helpers/a
       ../qa/workunits/ceph-helpers.sh:289: run_mon:  ceph-mon --id a --mkfs --mon-data=testdir/ceph-helpers/a --run-dir=testdir/ceph-helpers
       ceph-mon: mon.noname-a 127.0.0.1:7109/0 is local, renaming to mon.a
       ceph-mon: set fsid to 3f1b7cce-0c19-4c66-990c-9bbd1a39c039
       ceph-mon: created monfs at testdir/ceph-helpers/a for mon.a
       ../qa/workunits/ceph-helpers.sh:296: run_mon:  ceph-mon --id a --mon-osd-full-ratio=.99 --mon-data-avail-crit=1 --paxos-propose-interval=0.1 --osd-crush-chooseleaf-type=0 --osd-pool-default-erasure-code-directory=.libs --debug-mon 20 --debug-ms 20 --debug-paxos 20 --chdir= --mon-data=testdir/ceph-helpers/a '--log-file=testdir/ceph-helpers/$name.log' '--admin-socket=testdir/ceph-helpers/$cluster-$name.asok' --mon-cluster-log-file=testdir/ceph-helpers/log --run-dir=testdir/ceph-helpers '--pid-file=testdir/ceph-helpers/$name.pid'
       ../qa/workunits/ceph-helpers.sh:314: run_mon:  cat
       .../qa/workunits/ceph-helpers.sh:314: run_mon:  get_config mon a fsid
       .../qa/workunits/ceph-helpers.sh:653: get_config:  local daemon=mon
       .../qa/workunits/ceph-helpers.sh:654: get_config:  local id=a
       .../qa/workunits/ceph-helpers.sh:655: get_config:  local config=fsid
       .../qa/workunits/ceph-helpers.sh:657: get_config:  CEPH_ARGS=
       .../qa/workunits/ceph-helpers.sh:657: get_config:  ceph --format xml daemon testdir/ceph-helpers/ceph-mon.a.asok config get fsid
       .../qa/workunits/ceph-helpers.sh:660: get_config:  xmlstarlet sel -t -m //fsid -v . -n
       .../qa/workunits/ceph-helpers.sh:314: run_mon:  get_config mon a mon_host
       .../qa/workunits/ceph-helpers.sh:653: get_config:  local daemon=mon
       .../qa/workunits/ceph-helpers.sh:654: get_config:  local id=a
       .../qa/workunits/ceph-helpers.sh:655: get_config:  local config=mon_host
       .../qa/workunits/ceph-helpers.sh:657: get_config:  CEPH_ARGS=
       .../qa/workunits/ceph-helpers.sh:657: get_config:  ceph --format xml daemon testdir/ceph-helpers/ceph-mon.a.asok config get mon_host
       .../qa/workunits/ceph-helpers.sh:660: get_config:  xmlstarlet sel -t -m //mon_host -v . -n
       .../qa/workunits/ceph-helpers.sh:319: run_mon:  get_config mon a mon_initial_members
       .../qa/workunits/ceph-helpers.sh:653: get_config:  local daemon=mon
       .../qa/workunits/ceph-helpers.sh:654: get_config:  local id=a
       .../qa/workunits/ceph-helpers.sh:655: get_config:  local config=mon_initial_members
       .../qa/workunits/ceph-helpers.sh:657: get_config:  CEPH_ARGS=
       .../qa/workunits/ceph-helpers.sh:657: get_config:  ceph --format xml daemon testdir/ceph-helpers/ceph-mon.a.asok config get mon_initial_members
       .../qa/workunits/ceph-helpers.sh:660: get_config:  xmlstarlet sel -t -m //mon_initial_members -v . -n
       ../qa/workunits/ceph-helpers.sh:319: run_mon:  test -z ''
       ../qa/workunits/ceph-helpers.sh:320: run_mon:  ceph osd pool delete rbd rbd --yes-i-really-really-mean-it
       *** DEVELOPER MODE: setting PATH, PYTHONPATH and LD_LIBRARY_PATH ***
       pool 'rbd' removed
       ../qa/workunits/ceph-helpers.sh:321: run_mon:  ceph osd pool create rbd 4
       *** DEVELOPER MODE: setting PATH, PYTHONPATH and LD_LIBRARY_PATH ***
       pool 'rbd' created
       ../qa/workunits/ceph-helpers.sh:848: test_get_is_making_recovery_progress:  get_is_making_recovery_progress
       .../qa/workunits/ceph-helpers.sh:839: get_is_making_recovery_progress:  ceph --format xml status
       .../qa/workunits/ceph-helpers.sh:839: get_is_making_recovery_progress:  xmlstarlet sel -t -m //pgmap/recovering_keys_per_sec -v . -o ' ' -t -m //pgmap/recovering_bytes_per_sec -v . -o ' ' -t -m //pgmap/recovering_objects_per_sec -v .
       ../qa/workunits/ceph-helpers.sh:839: get_is_making_recovery_progress:  local progress=
       ../qa/workunits/ceph-helpers.sh:840: get_is_making_recovery_progress:  test -n ''
       ../qa/workunits/ceph-helpers.sh:849: test_get_is_making_recovery_progress:  teardown testdir/ceph-helpers
       ../qa/workunits/ceph-helpers.sh:117: teardown:  local dir=testdir/ceph-helpers
       ../qa/workunits/ceph-helpers.sh:118: teardown:  kill_daemons testdir/ceph-helpers
       .../qa/workunits/ceph-helpers.sh:192: kill_daemons:  shopt -q -o xtrace
       .../qa/workunits/ceph-helpers.sh:192: kill_daemons:  echo true
       ../qa/workunits/ceph-helpers.sh:192: kill_daemons:  local trace=true
       ../qa/workunits/ceph-helpers.sh:193: kill_daemons:  true
       ../qa/workunits/ceph-helpers.sh:193: kill_daemons:  shopt -u -o xtrace
       ../qa/workunits/ceph-helpers.sh:219: kill_daemons:  return 0
       .../qa/workunits/ceph-helpers.sh:119: teardown:  stat -f -c %T .
       ../qa/workunits/ceph-helpers.sh:119: teardown:  '[' ext2/ext3 == btrfs ']'
       ../qa/workunits/ceph-helpers.sh:122: teardown:  rm -fr testdir/ceph-helpers
       ../qa/workunits/ceph-helpers.sh:1198: run_tests:  for func in '$funcs'
       ../qa/workunits/ceph-helpers.sh:1199: run_tests:  test_get_last_scrub_stamp testdir/ceph-helpers
       ../qa/workunits/ceph-helpers.sh:932: test_get_last_scrub_stamp:  local dir=testdir/ceph-helpers
       ../qa/workunits/ceph-helpers.sh:934: test_get_last_scrub_stamp:  setup testdir/ceph-helpers
       ../qa/workunits/ceph-helpers.sh:92: setup:  local dir=testdir/ceph-helpers
       ../qa/workunits/ceph-helpers.sh:93: setup:  teardown testdir/ceph-helpers
       ../qa/workunits/ceph-helpers.sh:117: teardown:  local dir=testdir/ceph-helpers
       ../qa/workunits/ceph-helpers.sh:118: teardown:  kill_daemons testdir/ceph-helpers
       .../qa/workunits/ceph-helpers.sh:192: kill_daemons:  shopt -q -o xtrace
       .../qa/workunits/ceph-helpers.sh:192: kill_daemons:  echo true
       ../qa/workunits/ceph-helpers.sh:192: kill_daemons:  local trace=true
       ../qa/workunits/ceph-helpers.sh:193: kill_daemons:  true
       ../qa/workunits/ceph-helpers.sh:193: kill_daemons:  shopt -u -o xtrace
       ../qa/workunits/ceph-helpers.sh:219: kill_daemons:  return 0
       .../qa/workunits/ceph-helpers.sh:119: teardown:  stat -f -c %T .
       ../qa/workunits/ceph-helpers.sh:119: teardown:  '[' ext2/ext3 == btrfs ']'
       ../qa/workunits/ceph-helpers.sh:122: teardown:  rm -fr testdir/ceph-helpers
       ../qa/workunits/ceph-helpers.sh:94: setup:  mkdir -p testdir/ceph-helpers
       ../qa/workunits/ceph-helpers.sh:935: test_get_last_scrub_stamp:  run_mon testdir/ceph-helpers a --osd_pool_default_size=1
       ../qa/workunits/ceph-helpers.sh:282: run_mon:  local dir=testdir/ceph-helpers
       ../qa/workunits/ceph-helpers.sh:283: run_mon:  shift
       ../qa/workunits/ceph-helpers.sh:284: run_mon:  local id=a
       ../qa/workunits/ceph-helpers.sh:285: run_mon:  shift
       ../qa/workunits/ceph-helpers.sh:286: run_mon:  local data=testdir/ceph-helpers/a
       ../qa/workunits/ceph-helpers.sh:289: run_mon:  ceph-mon --id a --mkfs --mon-data=testdir/ceph-helpers/a --run-dir=testdir/ceph-helpers --osd_pool_default_size=1
       ceph-mon: mon.noname-a 127.0.0.1:7109/0 is local, renaming to mon.a
       ceph-mon: set fsid to 3f1b7cce-0c19-4c66-990c-9bbd1a39c039
       ceph-mon: created monfs at testdir/ceph-helpers/a for mon.a
       ../qa/workunits/ceph-helpers.sh:296: run_mon:  ceph-mon --id a --mon-osd-full-ratio=.99 --mon-data-avail-crit=1 --paxos-propose-interval=0.1 --osd-crush-chooseleaf-type=0 --osd-pool-default-erasure-code-directory=.libs --debug-mon 20 --debug-ms 20 --debug-paxos 20 --chdir= --mon-data=testdir/ceph-helpers/a '--log-file=testdir/ceph-helpers/$name.log' '--admin-socket=testdir/ceph-helpers/$cluster-$name.asok' --mon-cluster-log-file=testdir/ceph-helpers/log --run-dir=testdir/ceph-helpers '--pid-file=testdir/ceph-helpers/$name.pid' --osd_pool_default_size=1
       ../qa/workunits/ceph-helpers.sh:314: run_mon:  cat
       .../qa/workunits/ceph-helpers.sh:314: run_mon:  get_config mon a fsid
       .../qa/workunits/ceph-helpers.sh:653: get_config:  local daemon=mon
       .../qa/workunits/ceph-helpers.sh:654: get_config:  local id=a
       .../qa/workunits/ceph-helpers.sh:655: get_config:  local config=fsid
       .../qa/workunits/ceph-helpers.sh:657: get_config:  CEPH_ARGS=
       .../qa/workunits/ceph-helpers.sh:657: get_config:  ceph --format xml daemon testdir/ceph-helpers/ceph-mon.a.asok config get fsid
       .../qa/workunits/ceph-helpers.sh:660: get_config:  xmlstarlet sel -t -m //fsid -v . -n
       .../qa/workunits/ceph-helpers.sh:314: run_mon:  get_config mon a mon_host
       .../qa/workunits/ceph-helpers.sh:653: get_config:  local daemon=mon
       .../qa/workunits/ceph-helpers.sh:654: get_config:  local id=a
       .../qa/workunits/ceph-helpers.sh:655: get_config:  local config=mon_host
       .../qa/workunits/ceph-helpers.sh:657: get_config:  CEPH_ARGS=
       .../qa/workunits/ceph-helpers.sh:657: get_config:  ceph --format xml daemon testdir/ceph-helpers/ceph-mon.a.asok config get mon_host
       .../qa/workunits/ceph-helpers.sh:660: get_config:  xmlstarlet sel -t -m //mon_host -v . -n
       .../qa/workunits/ceph-helpers.sh:319: run_mon:  get_config mon a mon_initial_members
       .../qa/workunits/ceph-helpers.sh:653: get_config:  local daemon=mon
       .../qa/workunits/ceph-helpers.sh:654: get_config:  local id=a
       .../qa/workunits/ceph-helpers.sh:655: get_config:  local config=mon_initial_members
       .../qa/workunits/ceph-helpers.sh:657: get_config:  CEPH_ARGS=
       .../qa/workunits/ceph-helpers.sh:657: get_config:  ceph --format xml daemon testdir/ceph-helpers/ceph-mon.a.asok config get mon_initial_members
       .../qa/workunits/ceph-helpers.sh:660: get_config:  xmlstarlet sel -t -m //mon_initial_members -v . -n
       ../qa/workunits/ceph-helpers.sh:319: run_mon:  test -z ''
       ../qa/workunits/ceph-helpers.sh:320: run_mon:  ceph osd pool delete rbd rbd --yes-i-really-really-mean-it
       *** DEVELOPER MODE: setting PATH, PYTHONPATH and LD_LIBRARY_PATH ***
       pool 'rbd' removed
       ../qa/workunits/ceph-helpers.sh:321: run_mon:  ceph osd pool create rbd 4
       *** DEVELOPER MODE: setting PATH, PYTHONPATH and LD_LIBRARY_PATH ***
       pool 'rbd' created
       ../qa/workunits/ceph-helpers.sh:936: test_get_last_scrub_stamp:  run_osd testdir/ceph-helpers 0
       ../qa/workunits/ceph-helpers.sh:397: run_osd:  local dir=testdir/ceph-helpers
       ../qa/workunits/ceph-helpers.sh:398: run_osd:  shift
       ../qa/workunits/ceph-helpers.sh:399: run_osd:  local id=0
       ../qa/workunits/ceph-helpers.sh:400: run_osd:  shift
       ../qa/workunits/ceph-helpers.sh:401: run_osd:  local osd_data=testdir/ceph-helpers/0
       ../qa/workunits/ceph-helpers.sh:403: run_osd:  local ceph_disk_args
       ../qa/workunits/ceph-helpers.sh:404: run_osd:  ceph_disk_args+=' --statedir=testdir/ceph-helpers'
       ../qa/workunits/ceph-helpers.sh:405: run_osd:  ceph_disk_args+=' --sysconfdir=testdir/ceph-helpers'
       ../qa/workunits/ceph-helpers.sh:406: run_osd:  ceph_disk_args+=' --prepend-to-path='
       ../qa/workunits/ceph-helpers.sh:408: run_osd:  mkdir -p testdir/ceph-helpers/0
       ../qa/workunits/ceph-helpers.sh:409: run_osd:  ceph-disk --statedir=testdir/ceph-helpers --sysconfdir=testdir/ceph-helpers --prepend-to-path= prepare testdir/ceph-helpers/0
 -07-15 19:59:17.987425 7fd1627e57c0 -1 warning: unable to create /var/run/ceph: (13) Permission denied
 -07-15 19:59:18.005157 7fd3d9c647c0 -1 warning: unable to create /var/run/ceph: (13) Permission denied
 -07-15 19:59:18.019026 7f66c75f77c0 -1 warning: unable to create /var/run/ceph: (13) Permission denied
       ../qa/workunits/ceph-helpers.sh:412: run_osd:  activate_osd testdir/ceph-helpers 0
       ../qa/workunits/ceph-helpers.sh:479: activate_osd:  local dir=testdir/ceph-helpers
       ../qa/workunits/ceph-helpers.sh:480: activate_osd:  shift
       ../qa/workunits/ceph-helpers.sh:481: activate_osd:  local id=0
       ../qa/workunits/ceph-helpers.sh:482: activate_osd:  shift
       ../qa/workunits/ceph-helpers.sh:483: activate_osd:  local osd_data=testdir/ceph-helpers/0
       ../qa/workunits/ceph-helpers.sh:485: activate_osd:  local ceph_disk_args
       ../qa/workunits/ceph-helpers.sh:486: activate_osd:  ceph_disk_args+=' --statedir=testdir/ceph-helpers'
       ../qa/workunits/ceph-helpers.sh:487: activate_osd:  ceph_disk_args+=' --sysconfdir=testdir/ceph-helpers'
       ../qa/workunits/ceph-helpers.sh:488: activate_osd:  ceph_disk_args+=' --prepend-to-path='
       ../qa/workunits/ceph-helpers.sh:490: activate_osd:  local 'ceph_args=--fsid=3f1b7cce-0c19-4c66-990c-9bbd1a39c039 --auth-supported=none --mon-host=127.0.0.1:7109 '
       ../qa/workunits/ceph-helpers.sh:491: activate_osd:  ceph_args+=' --osd-backfill-full-ratio=.99'
       ../qa/workunits/ceph-helpers.sh:492: activate_osd:  ceph_args+=' --osd-failsafe-full-ratio=.99'
       ../qa/workunits/ceph-helpers.sh:493: activate_osd:  ceph_args+=' --osd-journal-size=100'
       ../qa/workunits/ceph-helpers.sh:494: activate_osd:  ceph_args+=' --osd-data=testdir/ceph-helpers/0'
       ../qa/workunits/ceph-helpers.sh:495: activate_osd:  ceph_args+=' --chdir='
       ../qa/workunits/ceph-helpers.sh:496: activate_osd:  ceph_args+=' --osd-pool-default-erasure-code-directory=.libs'
       ../qa/workunits/ceph-helpers.sh:497: activate_osd:  ceph_args+=' --osd-class-dir=.libs'
       ../qa/workunits/ceph-helpers.sh:498: activate_osd:  ceph_args+=' --run-dir=testdir/ceph-helpers'
       ../qa/workunits/ceph-helpers.sh:499: activate_osd:  ceph_args+=' --debug-osd=20'
       ../qa/workunits/ceph-helpers.sh:500: activate_osd:  ceph_args+=' --log-file=testdir/ceph-helpers/$name.log'
       ../qa/workunits/ceph-helpers.sh:501: activate_osd:  ceph_args+=' --pid-file=testdir/ceph-helpers/$name.pid'
       ../qa/workunits/ceph-helpers.sh:502: activate_osd:  ceph_args+=' '
       ../qa/workunits/ceph-helpers.sh:503: activate_osd:  ceph_args+=
       ../qa/workunits/ceph-helpers.sh:504: activate_osd:  mkdir -p testdir/ceph-helpers/0
       ../qa/workunits/ceph-helpers.sh:505: activate_osd:  CEPH_ARGS='--fsid=3f1b7cce-0c19-4c66-990c-9bbd1a39c039 --auth-supported=none --mon-host=127.0.0.1:7109  --osd-backfill-full-ratio=.99 --osd-failsafe-full-ratio=.99 --osd-journal-size=100 --osd-data=testdir/ceph-helpers/0 --chdir= --osd-pool-default-erasure-code-directory=.libs --osd-class-dir=.libs --run-dir=testdir/ceph-helpers --debug-osd=20 --log-file=testdir/ceph-helpers/$name.log --pid-file=testdir/ceph-helpers/$name.pid  '
       ../qa/workunits/ceph-helpers.sh:505: activate_osd:  ceph-disk --statedir=testdir/ceph-helpers --sysconfdir=testdir/ceph-helpers --prepend-to-path= activate --mark-init=none testdir/ceph-helpers/0
       *** DEVELOPER MODE: setting PATH, PYTHONPATH and LD_LIBRARY_PATH ***
       *** DEVELOPER MODE: setting PATH, PYTHONPATH and LD_LIBRARY_PATH ***
       got monmap epoch 1
 -07-15 19:59:20.397852 7fda274e67c0 -1 journal FileJournal::_open: disabling aio for non-block journal.  Use journal_force_aio to force use of aio anyway
 -07-15 19:59:20.504640 7fda274e67c0 -1 journal FileJournal::_open: disabling aio for non-block journal.  Use journal_force_aio to force use of aio anyway
 -07-15 19:59:20.505120 7fda274e67c0 -1 filestore(testdir/ceph-helpers/0) could not find -1/23c2fcde/osd_superblock/0 in index: (2) No such file or directory
 -07-15 19:59:20.605927 7fda274e67c0 -1 created object store testdir/ceph-helpers/0 journal testdir/ceph-helpers/0/journal for osd.0 fsid 3f1b7cce-0c19-4c66-990c-9bbd1a39c039
 -07-15 19:59:20.605993 7fda274e67c0 -1 auth: error reading file: testdir/ceph-helpers/0/keyring: can't open testdir/ceph-helpers/0/keyring: (2) No such file or directory
 -07-15 19:59:20.606171 7fda274e67c0 -1 created new key in keyring testdir/ceph-helpers/0/keyring
       *** DEVELOPER MODE: setting PATH, PYTHONPATH and LD_LIBRARY_PATH ***
       added key for osd.0
       starting osd.0 at :/0 osd_data testdir/ceph-helpers/0 testdir/ceph-helpers/0/journal
       .../qa/workunits/ceph-helpers.sh:510: activate_osd:  cat testdir/ceph-helpers/0/whoami
       ../qa/workunits/ceph-helpers.sh:510: activate_osd:  '[' 0 = 0 ']'
       ../qa/workunits/ceph-helpers.sh:512: activate_osd:  ceph osd crush create-or-move 0 1 root=default host=localhost
       *** DEVELOPER MODE: setting PATH, PYTHONPATH and LD_LIBRARY_PATH ***
       create-or-move updating item name 'osd.0' weight 1 at location {host=localhost,root=default} to crush map
       ../qa/workunits/ceph-helpers.sh:514: activate_osd:  wait_for_osd up 0
       ../qa/workunits/ceph-helpers.sh:550: wait_for_osd:  local state=up
       ../qa/workunits/ceph-helpers.sh:551: wait_for_osd:  local id=0
       ../qa/workunits/ceph-helpers.sh:553: wait_for_osd:  status=1
       ../qa/workunits/ceph-helpers.sh:554: wait_for_osd:  (( i=0 ))
       ../qa/workunits/ceph-helpers.sh:554: wait_for_osd:  (( i < 120 ))
       ../qa/workunits/ceph-helpers.sh:555: wait_for_osd:  ceph osd dump
       ../qa/workunits/ceph-helpers.sh:555: wait_for_osd:  grep 'osd.0 up'
       *** DEVELOPER MODE: setting PATH, PYTHONPATH and LD_LIBRARY_PATH ***
       osd.0 up   in  weight 1 up_from 5 up_thru 6 down_at 0 last_clean_interval [0,0) 127.0.0.1:6804/29933 127.0.0.1:6805/29933 127.0.0.1:6806/29933 127.0.0.1:6807/29933 exists,up d0bf3a6f-f6c7-47d6-be01-3b66da6c1aed
       ../qa/workunits/ceph-helpers.sh:558: wait_for_osd:  status=0
       ../qa/workunits/ceph-helpers.sh:559: wait_for_osd:  break
       ../qa/workunits/ceph-helpers.sh:562: wait_for_osd:  return 0
       ../qa/workunits/ceph-helpers.sh:937: test_get_last_scrub_stamp:  wait_for_clean
       ../qa/workunits/ceph-helpers.sh:981: wait_for_clean:  local status=1
       .../qa/workunits/ceph-helpers.sh:982: wait_for_clean:  get_num_active_clean
       .../qa/workunits/ceph-helpers.sh:863: get_num_active_clean:  local 'expression=('
       .../qa/workunits/ceph-helpers.sh:864: get_num_active_clean:  expression+='contains(.,'\''active'\'') and '
       .../qa/workunits/ceph-helpers.sh:865: get_num_active_clean:  expression+='contains(.,'\''clean'\'') and '
       .../qa/workunits/ceph-helpers.sh:866: get_num_active_clean:  expression+='not(contains(.,'\''stale'\''))'
       .../qa/workunits/ceph-helpers.sh:867: get_num_active_clean:  expression+=')'
       .../qa/workunits/ceph-helpers.sh:871: get_num_active_clean:  ceph --format xml pg dump pgs
       .../qa/workunits/ceph-helpers.sh:872: get_num_active_clean:  xmlstarlet sel -t -m '//pg_stat/state[(contains(.,'\''active'\'') and contains(.,'\''clean'\'') and not(contains(.,'\''stale'\'')))]' -v . -n
       .../qa/workunits/ceph-helpers.sh:873: get_num_active_clean:  grep -v '^$'
       .../qa/workunits/ceph-helpers.sh:873: get_num_active_clean:  wc -l
       ../qa/workunits/ceph-helpers.sh:982: wait_for_clean:  local num_active_clean=0
       ../qa/workunits/ceph-helpers.sh:983: wait_for_clean:  local cur_active_clean
       ../qa/workunits/ceph-helpers.sh:984: wait_for_clean:  local -i timer=0
       ../qa/workunits/ceph-helpers.sh:985: wait_for_clean:  is_clean
       .../qa/workunits/ceph-helpers.sh:952: is_clean:  get_num_pgs
       .../qa/workunits/ceph-helpers.sh:898: get_num_pgs:  ceph --format xml status
       .../qa/workunits/ceph-helpers.sh:899: get_num_pgs:  xmlstarlet sel -t -m //pgmap/num_pgs -v .
       ../qa/workunits/ceph-helpers.sh:952: is_clean:  num_pgs=4
       ../qa/workunits/ceph-helpers.sh:953: is_clean:  test 4 '!=' 0
       .../qa/workunits/ceph-helpers.sh:954: is_clean:  get_num_active_clean
       .../qa/workunits/ceph-helpers.sh:863: get_num_active_clean:  local 'expression=('
       .../qa/workunits/ceph-helpers.sh:864: get_num_active_clean:  expression+='contains(.,'\''active'\'') and '
       .../qa/workunits/ceph-helpers.sh:865: get_num_active_clean:  expression+='contains(.,'\''clean'\'') and '
       .../qa/workunits/ceph-helpers.sh:866: get_num_active_clean:  expression+='not(contains(.,'\''stale'\''))'
       .../qa/workunits/ceph-helpers.sh:867: get_num_active_clean:  expression+=')'
       .../qa/workunits/ceph-helpers.sh:871: get_num_active_clean:  ceph --format xml pg dump pgs
       .../qa/workunits/ceph-helpers.sh:872: get_num_active_clean:  xmlstarlet sel -t -m '//pg_stat/state[(contains(.,'\''active'\'') and contains(.,'\''clean'\'') and not(contains(.,'\''stale'\'')))]' -v . -n
       .../qa/workunits/ceph-helpers.sh:873: get_num_active_clean:  grep -v '^$'
       .../qa/workunits/ceph-helpers.sh:873: get_num_active_clean:  wc -l
       ../qa/workunits/ceph-helpers.sh:954: is_clean:  test 0 = 4
       ../qa/workunits/ceph-helpers.sh:954: is_clean:  return 1
       ../qa/workunits/ceph-helpers.sh:986: wait_for_clean:  get_is_making_recovery_progress
       .../qa/workunits/ceph-helpers.sh:839: get_is_making_recovery_progress:  ceph --format xml status
       .../qa/workunits/ceph-helpers.sh:839: get_is_making_recovery_progress:  xmlstarlet sel -t -m //pgmap/recovering_keys_per_sec -v . -o ' ' -t -m //pgmap/recovering_bytes_per_sec -v . -o ' ' -t -m //pgmap/recovering_objects_per_sec -v .
       ../qa/workunits/ceph-helpers.sh:839: get_is_making_recovery_progress:  local progress=
       ../qa/workunits/ceph-helpers.sh:840: get_is_making_recovery_progress:  test -n ''
       ../qa/workunits/ceph-helpers.sh:988: wait_for_clean:  ((  timer >= 120  ))
       .../qa/workunits/ceph-helpers.sh:993: wait_for_clean:  get_num_active_clean
       .../qa/workunits/ceph-helpers.sh:863: get_num_active_clean:  local 'expression=('
       .../qa/workunits/ceph-helpers.sh:864: get_num_active_clean:  expression+='contains(.,'\''active'\'') and '
       .../qa/workunits/ceph-helpers.sh:865: get_num_active_clean:  expression+='contains(.,'\''clean'\'') and '
       .../qa/workunits/ceph-helpers.sh:866: get_num_active_clean:  expression+='not(contains(.,'\''stale'\''))'
       .../qa/workunits/ceph-helpers.sh:867: get_num_active_clean:  expression+=')'
       .../qa/workunits/ceph-helpers.sh:871: get_num_active_clean:  ceph --format xml pg dump pgs
       .../qa/workunits/ceph-helpers.sh:872: get_num_active_clean:  xmlstarlet sel -t -m '//pg_stat/state[(contains(.,'\''active'\'') and contains(.,'\''clean'\'') and not(contains(.,'\''stale'\'')))]' -v . -n
       .../qa/workunits/ceph-helpers.sh:873: get_num_active_clean:  wc -l
       .../qa/workunits/ceph-helpers.sh:873: get_num_active_clean:  grep -v '^$'
       ../qa/workunits/ceph-helpers.sh:993: wait_for_clean:  cur_active_clean=0
       ../qa/workunits/ceph-helpers.sh:994: wait_for_clean:  test 0 '!=' 0
       ../qa/workunits/ceph-helpers.sh:998: wait_for_clean:  sleep 1
       .../qa/workunits/ceph-helpers.sh:999: wait_for_clean:  expr 0 + 1
       ../qa/workunits/ceph-helpers.sh:999: wait_for_clean:  timer=1
       ../qa/workunits/ceph-helpers.sh:985: wait_for_clean:  is_clean
       .../qa/workunits/ceph-helpers.sh:952: is_clean:  get_num_pgs
       .../qa/workunits/ceph-helpers.sh:898: get_num_pgs:  ceph --format xml status
       .../qa/workunits/ceph-helpers.sh:899: get_num_pgs:  xmlstarlet sel -t -m //pgmap/num_pgs -v .
       ../qa/workunits/ceph-helpers.sh:952: is_clean:  num_pgs=4
       ../qa/workunits/ceph-helpers.sh:953: is_clean:  test 4 '!=' 0
       .../qa/workunits/ceph-helpers.sh:954: is_clean:  get_num_active_clean
       .../qa/workunits/ceph-helpers.sh:863: get_num_active_clean:  local 'expression=('
       .../qa/workunits/ceph-helpers.sh:864: get_num_active_clean:  expression+='contains(.,'\''active'\'') and '
       .../qa/workunits/ceph-helpers.sh:865: get_num_active_clean:  expression+='contains(.,'\''clean'\'') and '
       .../qa/workunits/ceph-helpers.sh:866: get_num_active_clean:  expression+='not(contains(.,'\''stale'\''))'
       .../qa/workunits/ceph-helpers.sh:867: get_num_active_clean:  expression+=')'
       .../qa/workunits/ceph-helpers.sh:871: get_num_active_clean:  ceph --format xml pg dump pgs
       .../qa/workunits/ceph-helpers.sh:872: get_num_active_clean:  xmlstarlet sel -t -m '//pg_stat/state[(contains(.,'\''active'\'') and contains(.,'\''clean'\'') and not(contains(.,'\''stale'\'')))]' -v . -n
       .../qa/workunits/ceph-helpers.sh:873: get_num_active_clean:  grep -v '^$'
       .../qa/workunits/ceph-helpers.sh:873: get_num_active_clean:  wc -l
       ../qa/workunits/ceph-helpers.sh:954: is_clean:  test 0 = 4
       ../qa/workunits/ceph-helpers.sh:954: is_clean:  return 1
       ../qa/workunits/ceph-helpers.sh:986: wait_for_clean:  get_is_making_recovery_progress
       .../qa/workunits/ceph-helpers.sh:839: get_is_making_recovery_progress:  ceph --format xml status
       .../qa/workunits/ceph-helpers.sh:839: get_is_making_recovery_progress:  xmlstarlet sel -t -m //pgmap/recovering_keys_per_sec -v . -o ' ' -t -m //pgmap/recovering_bytes_per_sec -v . -o ' ' -t -m //pgmap/recovering_objects_per_sec -v .
       ../qa/workunits/ceph-helpers.sh:839: get_is_making_recovery_progress:  local progress=
       ../qa/workunits/ceph-helpers.sh:840: get_is_making_recovery_progress:  test -n ''
       ../qa/workunits/ceph-helpers.sh:988: wait_for_clean:  ((  timer >= 120  ))
       .../qa/workunits/ceph-helpers.sh:993: wait_for_clean:  get_num_active_clean
       .../qa/workunits/ceph-helpers.sh:863: get_num_active_clean:  local 'expression=('
       .../qa/workunits/ceph-helpers.sh:864: get_num_active_clean:  expression+='contains(.,'\''active'\'') and '
       .../qa/workunits/ceph-helpers.sh:865: get_num_active_clean:  expression+='contains(.,'\''clean'\'') and '
       .../qa/workunits/ceph-helpers.sh:866: get_num_active_clean:  expression+='not(contains(.,'\''stale'\''))'
       .../qa/workunits/ceph-helpers.sh:867: get_num_active_clean:  expression+=')'
       .../qa/workunits/ceph-helpers.sh:871: get_num_active_clean:  ceph --format xml pg dump pgs
       .../qa/workunits/ceph-helpers.sh:872: get_num_active_clean:  xmlstarlet sel -t -m '//pg_stat/state[(contains(.,'\''active'\'') and contains(.,'\''clean'\'') and not(contains(.,'\''stale'\'')))]' -v . -n
       .../qa/workunits/ceph-helpers.sh:873: get_num_active_clean:  grep -v '^$'
       .../qa/workunits/ceph-helpers.sh:873: get_num_active_clean:  wc -l
       ../qa/workunits/ceph-helpers.sh:993: wait_for_clean:  cur_active_clean=0
       ../qa/workunits/ceph-helpers.sh:994: wait_for_clean:  test 0 '!=' 0
       ../qa/workunits/ceph-helpers.sh:998: wait_for_clean:  sleep 1
       .../qa/workunits/ceph-helpers.sh:999: wait_for_clean:  expr 1 + 1
       ../qa/workunits/ceph-helpers.sh:999: wait_for_clean:  timer=2
       ../qa/workunits/ceph-helpers.sh:985: wait_for_clean:  is_clean
       .../qa/workunits/ceph-helpers.sh:952: is_clean:  get_num_pgs
       .../qa/workunits/ceph-helpers.sh:898: get_num_pgs:  ceph --format xml status
       .../qa/workunits/ceph-helpers.sh:899: get_num_pgs:  xmlstarlet sel -t -m //pgmap/num_pgs -v .
       ../qa/workunits/ceph-helpers.sh:952: is_clean:  num_pgs=4
       ../qa/workunits/ceph-helpers.sh:953: is_clean:  test 4 '!=' 0
       .../qa/workunits/ceph-helpers.sh:954: is_clean:  get_num_active_clean
       .../qa/workunits/ceph-helpers.sh:863: get_num_active_clean:  local 'expression=('
       .../qa/workunits/ceph-helpers.sh:864: get_num_active_clean:  expression+='contains(.,'\''active'\'') and '
       .../qa/workunits/ceph-helpers.sh:865: get_num_active_clean:  expression+='contains(.,'\''clean'\'') and '
       .../qa/workunits/ceph-helpers.sh:866: get_num_active_clean:  expression+='not(contains(.,'\''stale'\''))'
       .../qa/workunits/ceph-helpers.sh:867: get_num_active_clean:  expression+=')'
       .../qa/workunits/ceph-helpers.sh:871: get_num_active_clean:  ceph --format xml pg dump pgs
       .../qa/workunits/ceph-helpers.sh:872: get_num_active_clean:  xmlstarlet sel -t -m '//pg_stat/state[(contains(.,'\''active'\'') and contains(.,'\''clean'\'') and not(contains(.,'\''stale'\'')))]' -v . -n
       .../qa/workunits/ceph-helpers.sh:873: get_num_active_clean:  grep -v '^$'
       .../qa/workunits/ceph-helpers.sh:873: get_num_active_clean:  wc -l
       ../qa/workunits/ceph-helpers.sh:954: is_clean:  test 0 = 4
       ../qa/workunits/ceph-helpers.sh:954: is_clean:  return 1
       ../qa/workunits/ceph-helpers.sh:986: wait_for_clean:  get_is_making_recovery_progress
       .../qa/workunits/ceph-helpers.sh:839: get_is_making_recovery_progress:  ceph --format xml status
       .../qa/workunits/ceph-helpers.sh:839: get_is_making_recovery_progress:  xmlstarlet sel -t -m //pgmap/recovering_keys_per_sec -v . -o ' ' -t -m //pgmap/recovering_bytes_per_sec -v . -o ' ' -t -m //pgmap/recovering_objects_per_sec -v .
       ../qa/workunits/ceph-helpers.sh:839: get_is_making_recovery_progress:  local progress=
       ../qa/workunits/ceph-helpers.sh:840: get_is_making_recovery_progress:  test -n ''
       ../qa/workunits/ceph-helpers.sh:988: wait_for_clean:  ((  timer >= 120  ))
       .../qa/workunits/ceph-helpers.sh:993: wait_for_clean:  get_num_active_clean
       .../qa/workunits/ceph-helpers.sh:863: get_num_active_clean:  local 'expression=('
       .../qa/workunits/ceph-helpers.sh:864: get_num_active_clean:  expression+='contains(.,'\''active'\'') and '
       .../qa/workunits/ceph-helpers.sh:865: get_num_active_clean:  expression+='contains(.,'\''clean'\'') and '
       .../qa/workunits/ceph-helpers.sh:866: get_num_active_clean:  expression+='not(contains(.,'\''stale'\''))'
       .../qa/workunits/ceph-helpers.sh:867: get_num_active_clean:  expression+=')'
       .../qa/workunits/ceph-helpers.sh:871: get_num_active_clean:  ceph --format xml pg dump pgs
       .../qa/workunits/ceph-helpers.sh:872: get_num_active_clean:  xmlstarlet sel -t -m '//pg_stat/state[(contains(.,'\''active'\'') and contains(.,'\''clean'\'') and not(contains(.,'\''stale'\'')))]' -v . -n
       .../qa/workunits/ceph-helpers.sh:873: get_num_active_clean:  grep -v '^$'
       .../qa/workunits/ceph-helpers.sh:873: get_num_active_clean:  wc -l
       ../qa/workunits/ceph-helpers.sh:993: wait_for_clean:  cur_active_clean=0
       ../qa/workunits/ceph-helpers.sh:994: wait_for_clean:  test 0 '!=' 0
       ../qa/workunits/ceph-helpers.sh:998: wait_for_clean:  sleep 1
       .../qa/workunits/ceph-helpers.sh:999: wait_for_clean:  expr 2 + 1
       ../qa/workunits/ceph-helpers.sh:999: wait_for_clean:  timer=3
       ../qa/workunits/ceph-helpers.sh:985: wait_for_clean:  is_clean
       .../qa/workunits/ceph-helpers.sh:952: is_clean:  get_num_pgs
       .../qa/workunits/ceph-helpers.sh:898: get_num_pgs:  ceph --format xml status
       .../qa/workunits/ceph-helpers.sh:899: get_num_pgs:  xmlstarlet sel -t -m //pgmap/num_pgs -v .
       ../qa/workunits/ceph-helpers.sh:952: is_clean:  num_pgs=4
       ../qa/workunits/ceph-helpers.sh:953: is_clean:  test 4 '!=' 0
       .../qa/workunits/ceph-helpers.sh:954: is_clean:  get_num_active_clean
       .../qa/workunits/ceph-helpers.sh:863: get_num_active_clean:  local 'expression=('
       .../qa/workunits/ceph-helpers.sh:864: get_num_active_clean:  expression+='contains(.,'\''active'\'') and '
       .../qa/workunits/ceph-helpers.sh:865: get_num_active_clean:  expression+='contains(.,'\''clean'\'') and '
       .../qa/workunits/ceph-helpers.sh:866: get_num_active_clean:  expression+='not(contains(.,'\''stale'\''))'
       .../qa/workunits/ceph-helpers.sh:867: get_num_active_clean:  expression+=')'
       .../qa/workunits/ceph-helpers.sh:871: get_num_active_clean:  ceph --format xml pg dump pgs
       .../qa/workunits/ceph-helpers.sh:872: get_num_active_clean:  xmlstarlet sel -t -m '//pg_stat/state[(contains(.,'\''active'\'') and contains(.,'\''clean'\'') and not(contains(.,'\''stale'\'')))]' -v . -n
       .../qa/workunits/ceph-helpers.sh:873: get_num_active_clean:  grep -v '^$'
       .../qa/workunits/ceph-helpers.sh:873: get_num_active_clean:  wc -l
       ../qa/workunits/ceph-helpers.sh:954: is_clean:  test 0 = 4
       ../qa/workunits/ceph-helpers.sh:954: is_clean:  return 1
       ../qa/workunits/ceph-helpers.sh:986: wait_for_clean:  get_is_making_recovery_progress
       .../qa/workunits/ceph-helpers.sh:839: get_is_making_recovery_progress:  ceph --format xml status
       .../qa/workunits/ceph-helpers.sh:839: get_is_making_recovery_progress:  xmlstarlet sel -t -m //pgmap/recovering_keys_per_sec -v . -o ' ' -t -m //pgmap/recovering_bytes_per_sec -v . -o ' ' -t -m //pgmap/recovering_objects_per_sec -v .
       ../qa/workunits/ceph-helpers.sh:839: get_is_making_recovery_progress:  local progress=
       ../qa/workunits/ceph-helpers.sh:840: get_is_making_recovery_progress:  test -n ''
       ../qa/workunits/ceph-helpers.sh:988: wait_for_clean:  ((  timer >= 120  ))
       .../qa/workunits/ceph-helpers.sh:993: wait_for_clean:  get_num_active_clean
       .../qa/workunits/ceph-helpers.sh:863: get_num_active_clean:  local 'expression=('
       .../qa/workunits/ceph-helpers.sh:864: get_num_active_clean:  expression+='contains(.,'\''active'\'') and '
       .../qa/workunits/ceph-helpers.sh:865: get_num_active_clean:  expression+='contains(.,'\''clean'\'') and '
       .../qa/workunits/ceph-helpers.sh:866: get_num_active_clean:  expression+='not(contains(.,'\''stale'\''))'
       .../qa/workunits/ceph-helpers.sh:867: get_num_active_clean:  expression+=')'
       .../qa/workunits/ceph-helpers.sh:871: get_num_active_clean:  ceph --format xml pg dump pgs
       .../qa/workunits/ceph-helpers.sh:872: get_num_active_clean:  xmlstarlet sel -t -m '//pg_stat/state[(contains(.,'\''active'\'') and contains(.,'\''clean'\'') and not(contains(.,'\''stale'\'')))]' -v . -n
       .../qa/workunits/ceph-helpers.sh:873: get_num_active_clean:  grep -v '^$'
       .../qa/workunits/ceph-helpers.sh:873: get_num_active_clean:  wc -l
       ../qa/workunits/ceph-helpers.sh:993: wait_for_clean:  cur_active_clean=0
       ../qa/workunits/ceph-helpers.sh:994: wait_for_clean:  test 0 '!=' 0
       ../qa/workunits/ceph-helpers.sh:998: wait_for_clean:  sleep 1
       .../qa/workunits/ceph-helpers.sh:999: wait_for_clean:  expr 3 + 1
       ../qa/workunits/ceph-helpers.sh:999: wait_for_clean:  timer=4
       ../qa/workunits/ceph-helpers.sh:985: wait_for_clean:  is_clean
       .../qa/workunits/ceph-helpers.sh:952: is_clean:  get_num_pgs
       .../qa/workunits/ceph-helpers.sh:898: get_num_pgs:  ceph --format xml status
       .../qa/workunits/ceph-helpers.sh:899: get_num_pgs:  xmlstarlet sel -t -m //pgmap/num_pgs -v .
       ../qa/workunits/ceph-helpers.sh:952: is_clean:  num_pgs=4
       ../qa/workunits/ceph-helpers.sh:953: is_clean:  test 4 '!=' 0
       .../qa/workunits/ceph-helpers.sh:954: is_clean:  get_num_active_clean
       .../qa/workunits/ceph-helpers.sh:863: get_num_active_clean:  local 'expression=('
       .../qa/workunits/ceph-helpers.sh:864: get_num_active_clean:  expression+='contains(.,'\''active'\'') and '
       .../qa/workunits/ceph-helpers.sh:865: get_num_active_clean:  expression+='contains(.,'\''clean'\'') and '
       .../qa/workunits/ceph-helpers.sh:866: get_num_active_clean:  expression+='not(contains(.,'\''stale'\''))'
       .../qa/workunits/ceph-helpers.sh:867: get_num_active_clean:  expression+=')'
       .../qa/workunits/ceph-helpers.sh:871: get_num_active_clean:  ceph --format xml pg dump pgs
       .../qa/workunits/ceph-helpers.sh:872: get_num_active_clean:  xmlstarlet sel -t -m '//pg_stat/state[(contains(.,'\''active'\'') and contains(.,'\''clean'\'') and not(contains(.,'\''stale'\'')))]' -v . -n
       .../qa/workunits/ceph-helpers.sh:873: get_num_active_clean:  grep -v '^$'
       .../qa/workunits/ceph-helpers.sh:873: get_num_active_clean:  wc -l
       ../qa/workunits/ceph-helpers.sh:954: is_clean:  test 0 = 4
       ../qa/workunits/ceph-helpers.sh:954: is_clean:  return 1
       ../qa/workunits/ceph-helpers.sh:986: wait_for_clean:  get_is_making_recovery_progress
       .../qa/workunits/ceph-helpers.sh:839: get_is_making_recovery_progress:  ceph --format xml status
       .../qa/workunits/ceph-helpers.sh:839: get_is_making_recovery_progress:  xmlstarlet sel -t -m //pgmap/recovering_keys_per_sec -v . -o ' ' -t -m //pgmap/recovering_bytes_per_sec -v . -o ' ' -t -m //pgmap/recovering_objects_per_sec -v .
       ../qa/workunits/ceph-helpers.sh:839: get_is_making_recovery_progress:  local progress=
       ../qa/workunits/ceph-helpers.sh:840: get_is_making_recovery_progress:  test -n ''
       ../qa/workunits/ceph-helpers.sh:988: wait_for_clean:  ((  timer >= 120  ))
       .../qa/workunits/ceph-helpers.sh:993: wait_for_clean:  get_num_active_clean
       .../qa/workunits/ceph-helpers.sh:863: get_num_active_clean:  local 'expression=('
       .../qa/workunits/ceph-helpers.sh:864: get_num_active_clean:  expression+='contains(.,'\''active'\'') and '
       .../qa/workunits/ceph-helpers.sh:865: get_num_active_clean:  expression+='contains(.,'\''clean'\'') and '
       .../qa/workunits/ceph-helpers.sh:866: get_num_active_clean:  expression+='not(contains(.,'\''stale'\''))'
       .../qa/workunits/ceph-helpers.sh:867: get_num_active_clean:  expression+=')'
       .../qa/workunits/ceph-helpers.sh:871: get_num_active_clean:  ceph --format xml pg dump pgs
       .../qa/workunits/ceph-helpers.sh:872: get_num_active_clean:  xmlstarlet sel -t -m '//pg_stat/state[(contains(.,'\''active'\'') and contains(.,'\''clean'\'') and not(contains(.,'\''stale'\'')))]' -v . -n
       .../qa/workunits/ceph-helpers.sh:873: get_num_active_clean:  grep -v '^$'
       .../qa/workunits/ceph-helpers.sh:873: get_num_active_clean:  wc -l
       ../qa/workunits/ceph-helpers.sh:993: wait_for_clean:  cur_active_clean=0
       ../qa/workunits/ceph-helpers.sh:994: wait_for_clean:  test 0 '!=' 0
       ../qa/workunits/ceph-helpers.sh:998: wait_for_clean:  sleep 1
       .../qa/workunits/ceph-helpers.sh:999: wait_for_clean:  expr 4 + 1
       ../qa/workunits/ceph-helpers.sh:999: wait_for_clean:  timer=5
       ../qa/workunits/ceph-helpers.sh:985: wait_for_clean:  is_clean
       .../qa/workunits/ceph-helpers.sh:952: is_clean:  get_num_pgs
       .../qa/workunits/ceph-helpers.sh:898: get_num_pgs:  ceph --format xml status
       .../qa/workunits/ceph-helpers.sh:899: get_num_pgs:  xmlstarlet sel -t -m //pgmap/num_pgs -v .
       ../qa/workunits/ceph-helpers.sh:952: is_clean:  num_pgs=4
       ../qa/workunits/ceph-helpers.sh:953: is_clean:  test 4 '!=' 0
       .../qa/workunits/ceph-helpers.sh:954: is_clean:  get_num_active_clean
       .../qa/workunits/ceph-helpers.sh:863: get_num_active_clean:  local 'expression=('
       .../qa/workunits/ceph-helpers.sh:864: get_num_active_clean:  expression+='contains(.,'\''active'\'') and '
       .../qa/workunits/ceph-helpers.sh:865: get_num_active_clean:  expression+='contains(.,'\''clean'\'') and '
       .../qa/workunits/ceph-helpers.sh:866: get_num_active_clean:  expression+='not(contains(.,'\''stale'\''))'
       .../qa/workunits/ceph-helpers.sh:867: get_num_active_clean:  expression+=')'
       .../qa/workunits/ceph-helpers.sh:871: get_num_active_clean:  ceph --format xml pg dump pgs
       .../qa/workunits/ceph-helpers.sh:872: get_num_active_clean:  xmlstarlet sel -t -m '//pg_stat/state[(contains(.,'\''active'\'') and contains(.,'\''clean'\'') and not(contains(.,'\''stale'\'')))]' -v . -n
       .../qa/workunits/ceph-helpers.sh:873: get_num_active_clean:  wc -l
       .../qa/workunits/ceph-helpers.sh:873: get_num_active_clean:  grep -v '^$'
       ../qa/workunits/ceph-helpers.sh:954: is_clean:  test 0 = 4
       ../qa/workunits/ceph-helpers.sh:954: is_clean:  return 1
       ../qa/workunits/ceph-helpers.sh:986: wait_for_clean:  get_is_making_recovery_progress
       .../qa/workunits/ceph-helpers.sh:839: get_is_making_recovery_progress:  ceph --format xml status
       .../qa/workunits/ceph-helpers.sh:839: get_is_making_recovery_progress:  xmlstarlet sel -t -m //pgmap/recovering_keys_per_sec -v . -o ' ' -t -m //pgmap/recovering_bytes_per_sec -v . -o ' ' -t -m //pgmap/recovering_objects_per_sec -v .
       ../qa/workunits/ceph-helpers.sh:839: get_is_making_recovery_progress:  local progress=
       ../qa/workunits/ceph-helpers.sh:840: get_is_making_recovery_progress:  test -n ''
       ../qa/workunits/ceph-helpers.sh:988: wait_for_clean:  ((  timer >= 120  ))
       .../qa/workunits/ceph-helpers.sh:993: wait_for_clean:  get_num_active_clean
       .../qa/workunits/ceph-helpers.sh:863: get_num_active_clean:  local 'expression=('
       .../qa/workunits/ceph-helpers.sh:864: get_num_active_clean:  expression+='contains(.,'\''active'\'') and '
       .../qa/workunits/ceph-helpers.sh:865: get_num_active_clean:  expression+='contains(.,'\''clean'\'') and '
       .../qa/workunits/ceph-helpers.sh:866: get_num_active_clean:  expression+='not(contains(.,'\''stale'\''))'
       .../qa/workunits/ceph-helpers.sh:867: get_num_active_clean:  expression+=')'
       .../qa/workunits/ceph-helpers.sh:871: get_num_active_clean:  ceph --format xml pg dump pgs
       .../qa/workunits/ceph-helpers.sh:872: get_num_active_clean:  xmlstarlet sel -t -m '//pg_stat/state[(contains(.,'\''active'\'') and contains(.,'\''clean'\'') and not(contains(.,'\''stale'\'')))]' -v . -n
       .../qa/workunits/ceph-helpers.sh:873: get_num_active_clean:  grep -v '^$'
       .../qa/workunits/ceph-helpers.sh:873: get_num_active_clean:  wc -l
       ../qa/workunits/ceph-helpers.sh:993: wait_for_clean:  cur_active_clean=0
       ../qa/workunits/ceph-helpers.sh:994: wait_for_clean:  test 0 '!=' 0
       ../qa/workunits/ceph-helpers.sh:998: wait_for_clean:  sleep 1
       .../qa/workunits/ceph-helpers.sh:999: wait_for_clean:  expr 5 + 1
       ../qa/workunits/ceph-helpers.sh:999: wait_for_clean:  timer=6
       ../qa/workunits/ceph-helpers.sh:985: wait_for_clean:  is_clean
       .../qa/workunits/ceph-helpers.sh:952: is_clean:  get_num_pgs
       .../qa/workunits/ceph-helpers.sh:898: get_num_pgs:  ceph --format xml status
       .../qa/workunits/ceph-helpers.sh:899: get_num_pgs:  xmlstarlet sel -t -m //pgmap/num_pgs -v .
       ../qa/workunits/ceph-helpers.sh:952: is_clean:  num_pgs=4
       ../qa/workunits/ceph-helpers.sh:953: is_clean:  test 4 '!=' 0
       .../qa/workunits/ceph-helpers.sh:954: is_clean:  get_num_active_clean
       .../qa/workunits/ceph-helpers.sh:863: get_num_active_clean:  local 'expression=('
       .../qa/workunits/ceph-helpers.sh:864: get_num_active_clean:  expression+='contains(.,'\''active'\'') and '
       .../qa/workunits/ceph-helpers.sh:865: get_num_active_clean:  expression+='contains(.,'\''clean'\'') and '
       .../qa/workunits/ceph-helpers.sh:866: get_num_active_clean:  expression+='not(contains(.,'\''stale'\''))'
       .../qa/workunits/ceph-helpers.sh:867: get_num_active_clean:  expression+=')'
       .../qa/workunits/ceph-helpers.sh:871: get_num_active_clean:  ceph --format xml pg dump pgs
       .../qa/workunits/ceph-helpers.sh:873: get_num_active_clean:  grep -v '^$'
       .../qa/workunits/ceph-helpers.sh:872: get_num_active_clean:  xmlstarlet sel -t -m '//pg_stat/state[(contains(.,'\''active'\'') and contains(.,'\''clean'\'') and not(contains(.,'\''stale'\'')))]' -v . -n
       .../qa/workunits/ceph-helpers.sh:873: get_num_active_clean:  wc -l
       ../qa/workunits/ceph-helpers.sh:954: is_clean:  test 0 = 4
       ../qa/workunits/ceph-helpers.sh:954: is_clean:  return 1
       ../qa/workunits/ceph-helpers.sh:986: wait_for_clean:  get_is_making_recovery_progress
       .../qa/workunits/ceph-helpers.sh:839: get_is_making_recovery_progress:  ceph --format xml status
       .../qa/workunits/ceph-helpers.sh:839: get_is_making_recovery_progress:  xmlstarlet sel -t -m //pgmap/recovering_keys_per_sec -v . -o ' ' -t -m //pgmap/recovering_bytes_per_sec -v . -o ' ' -t -m //pgmap/recovering_objects_per_sec -v .
       ../qa/workunits/ceph-helpers.sh:839: get_is_making_recovery_progress:  local progress=
       ../qa/workunits/ceph-helpers.sh:840: get_is_making_recovery_progress:  test -n ''
       ../qa/workunits/ceph-helpers.sh:988: wait_for_clean:  ((  timer >= 120  ))
       .../qa/workunits/ceph-helpers.sh:993: wait_for_clean:  get_num_active_clean
       .../qa/workunits/ceph-helpers.sh:863: get_num_active_clean:  local 'expression=('
       .../qa/workunits/ceph-helpers.sh:864: get_num_active_clean:  expression+='contains(.,'\''active'\'') and '
       .../qa/workunits/ceph-helpers.sh:865: get_num_active_clean:  expression+='contains(.,'\''clean'\'') and '
       .../qa/workunits/ceph-helpers.sh:866: get_num_active_clean:  expression+='not(contains(.,'\''stale'\''))'
       .../qa/workunits/ceph-helpers.sh:867: get_num_active_clean:  expression+=')'
       .../qa/workunits/ceph-helpers.sh:871: get_num_active_clean:  ceph --format xml pg dump pgs
       .../qa/workunits/ceph-helpers.sh:872: get_num_active_clean:  xmlstarlet sel -t -m '//pg_stat/state[(contains(.,'\''active'\'') and contains(.,'\''clean'\'') and not(contains(.,'\''stale'\'')))]' -v . -n
       .../qa/workunits/ceph-helpers.sh:873: get_num_active_clean:  grep -v '^$'
       .../qa/workunits/ceph-helpers.sh:873: get_num_active_clean:  wc -l
       ../qa/workunits/ceph-helpers.sh:993: wait_for_clean:  cur_active_clean=0
       ../qa/workunits/ceph-helpers.sh:994: wait_for_clean:  test 0 '!=' 0
       ../qa/workunits/ceph-helpers.sh:998: wait_for_clean:  sleep 1
       .../qa/workunits/ceph-helpers.sh:999: wait_for_clean:  expr 6 + 1
       ../qa/workunits/ceph-helpers.sh:999: wait_for_clean:  timer=7
       ../qa/workunits/ceph-helpers.sh:985: wait_for_clean:  is_clean
       .../qa/workunits/ceph-helpers.sh:952: is_clean:  get_num_pgs
       .../qa/workunits/ceph-helpers.sh:898: get_num_pgs:  ceph --format xml status
       .../qa/workunits/ceph-helpers.sh:899: get_num_pgs:  xmlstarlet sel -t -m //pgmap/num_pgs -v .
       ../qa/workunits/ceph-helpers.sh:952: is_clean:  num_pgs=4
       ../qa/workunits/ceph-helpers.sh:953: is_clean:  test 4 '!=' 0
       .../qa/workunits/ceph-helpers.sh:954: is_clean:  get_num_active_clean
       .../qa/workunits/ceph-helpers.sh:863: get_num_active_clean:  local 'expression=('
       .../qa/workunits/ceph-helpers.sh:864: get_num_active_clean:  expression+='contains(.,'\''active'\'') and '
       .../qa/workunits/ceph-helpers.sh:865: get_num_active_clean:  expression+='contains(.,'\''clean'\'') and '
       .../qa/workunits/ceph-helpers.sh:866: get_num_active_clean:  expression+='not(contains(.,'\''stale'\''))'
       .../qa/workunits/ceph-helpers.sh:867: get_num_active_clean:  expression+=')'
       .../qa/workunits/ceph-helpers.sh:871: get_num_active_clean:  ceph --format xml pg dump pgs
       .../qa/workunits/ceph-helpers.sh:872: get_num_active_clean:  xmlstarlet sel -t -m '//pg_stat/state[(contains(.,'\''active'\'') and contains(.,'\''clean'\'') and not(contains(.,'\''stale'\'')))]' -v . -n
       .../qa/workunits/ceph-helpers.sh:873: get_num_active_clean:  grep -v '^$'
       .../qa/workunits/ceph-helpers.sh:873: get_num_active_clean:  wc -l
       ../qa/workunits/ceph-helpers.sh:954: is_clean:  test 0 = 4
       ../qa/workunits/ceph-helpers.sh:954: is_clean:  return 1
       ../qa/workunits/ceph-helpers.sh:986: wait_for_clean:  get_is_making_recovery_progress
       .../qa/workunits/ceph-helpers.sh:839: get_is_making_recovery_progress:  ceph --format xml status
       .../qa/workunits/ceph-helpers.sh:839: get_is_making_recovery_progress:  xmlstarlet sel -t -m //pgmap/recovering_keys_per_sec -v . -o ' ' -t -m //pgmap/recovering_bytes_per_sec -v . -o ' ' -t -m //pgmap/recovering_objects_per_sec -v .
       ../qa/workunits/ceph-helpers.sh:839: get_is_making_recovery_progress:  local progress=
       ../qa/workunits/ceph-helpers.sh:840: get_is_making_recovery_progress:  test -n ''
       ../qa/workunits/ceph-helpers.sh:988: wait_for_clean:  ((  timer >= 120  ))
       .../qa/workunits/ceph-helpers.sh:993: wait_for_clean:  get_num_active_clean
       .../qa/workunits/ceph-helpers.sh:863: get_num_active_clean:  local 'expression=('
       .../qa/workunits/ceph-helpers.sh:864: get_num_active_clean:  expression+='contains(.,'\''active'\'') and '
       .../qa/workunits/ceph-helpers.sh:865: get_num_active_clean:  expression+='contains(.,'\''clean'\'') and '
       .../qa/workunits/ceph-helpers.sh:866: get_num_active_clean:  expression+='not(contains(.,'\''stale'\''))'
       .../qa/workunits/ceph-helpers.sh:867: get_num_active_clean:  expression+=')'
       .../qa/workunits/ceph-helpers.sh:871: get_num_active_clean:  ceph --format xml pg dump pgs
       .../qa/workunits/ceph-helpers.sh:872: get_num_active_clean:  xmlstarlet sel -t -m '//pg_stat/state[(contains(.,'\''active'\'') and contains(.,'\''clean'\'') and not(contains(.,'\''stale'\'')))]' -v . -n
       .../qa/workunits/ceph-helpers.sh:873: get_num_active_clean:  grep -v '^$'
       .../qa/workunits/ceph-helpers.sh:873: get_num_active_clean:  wc -l
       ../qa/workunits/ceph-helpers.sh:993: wait_for_clean:  cur_active_clean=0
       ../qa/workunits/ceph-helpers.sh:994: wait_for_clean:  test 0 '!=' 0
       ../qa/workunits/ceph-helpers.sh:998: wait_for_clean:  sleep 1
       .../qa/workunits/ceph-helpers.sh:999: wait_for_clean:  expr 7 + 1
       ../qa/workunits/ceph-helpers.sh:999: wait_for_clean:  timer=8
       ../qa/workunits/ceph-helpers.sh:985: wait_for_clean:  is_clean
       .../qa/workunits/ceph-helpers.sh:952: is_clean:  get_num_pgs
       .../qa/workunits/ceph-helpers.sh:898: get_num_pgs:  ceph --format xml status
       .../qa/workunits/ceph-helpers.sh:899: get_num_pgs:  xmlstarlet sel -t -m //pgmap/num_pgs -v .
       ../qa/workunits/ceph-helpers.sh:952: is_clean:  num_pgs=4
       ../qa/workunits/ceph-helpers.sh:953: is_clean:  test 4 '!=' 0
       .../qa/workunits/ceph-helpers.sh:954: is_clean:  get_num_active_clean
       .../qa/workunits/ceph-helpers.sh:863: get_num_active_clean:  local 'expression=('
       .../qa/workunits/ceph-helpers.sh:864: get_num_active_clean:  expression+='contains(.,'\''active'\'') and '
       .../qa/workunits/ceph-helpers.sh:865: get_num_active_clean:  expression+='contains(.,'\''clean'\'') and '
       .../qa/workunits/ceph-helpers.sh:866: get_num_active_clean:  expression+='not(contains(.,'\''stale'\''))'
       .../qa/workunits/ceph-helpers.sh:867: get_num_active_clean:  expression+=')'
       .../qa/workunits/ceph-helpers.sh:871: get_num_active_clean:  ceph --format xml pg dump pgs
       .../qa/workunits/ceph-helpers.sh:872: get_num_active_clean:  xmlstarlet sel -t -m '//pg_stat/state[(contains(.,'\''active'\'') and contains(.,'\''clean'\'') and not(contains(.,'\''stale'\'')))]' -v . -n
       .../qa/workunits/ceph-helpers.sh:873: get_num_active_clean:  wc -l
       .../qa/workunits/ceph-helpers.sh:873: get_num_active_clean:  grep -v '^$'
       ../qa/workunits/ceph-helpers.sh:954: is_clean:  test 0 = 4
       ../qa/workunits/ceph-helpers.sh:954: is_clean:  return 1
       ../qa/workunits/ceph-helpers.sh:986: wait_for_clean:  get_is_making_recovery_progress
       .../qa/workunits/ceph-helpers.sh:839: get_is_making_recovery_progress:  ceph --format xml status
       .../qa/workunits/ceph-helpers.sh:839: get_is_making_recovery_progress:  xmlstarlet sel -t -m //pgmap/recovering_keys_per_sec -v . -o ' ' -t -m //pgmap/recovering_bytes_per_sec -v . -o ' ' -t -m //pgmap/recovering_objects_per_sec -v .
       ../qa/workunits/ceph-helpers.sh:839: get_is_making_recovery_progress:  local progress=
       ../qa/workunits/ceph-helpers.sh:840: get_is_making_recovery_progress:  test -n ''
       ../qa/workunits/ceph-helpers.sh:988: wait_for_clean:  ((  timer >= 120  ))
       .../qa/workunits/ceph-helpers.sh:993: wait_for_clean:  get_num_active_clean
       .../qa/workunits/ceph-helpers.sh:863: get_num_active_clean:  local 'expression=('
       .../qa/workunits/ceph-helpers.sh:864: get_num_active_clean:  expression+='contains(.,'\''active'\'') and '
       .../qa/workunits/ceph-helpers.sh:865: get_num_active_clean:  expression+='contains(.,'\''clean'\'') and '
       .../qa/workunits/ceph-helpers.sh:866: get_num_active_clean:  expression+='not(contains(.,'\''stale'\''))'
       .../qa/workunits/ceph-helpers.sh:867: get_num_active_clean:  expression+=')'
       .../qa/workunits/ceph-helpers.sh:871: get_num_active_clean:  ceph --format xml pg dump pgs
       .../qa/workunits/ceph-helpers.sh:872: get_num_active_clean:  xmlstarlet sel -t -m '//pg_stat/state[(contains(.,'\''active'\'') and contains(.,'\''clean'\'') and not(contains(.,'\''stale'\'')))]' -v . -n
       .../qa/workunits/ceph-helpers.sh:873: get_num_active_clean:  grep -v '^$'
       .../qa/workunits/ceph-helpers.sh:873: get_num_active_clean:  wc -l
       ../qa/workunits/ceph-helpers.sh:993: wait_for_clean:  cur_active_clean=0
       ../qa/workunits/ceph-helpers.sh:994: wait_for_clean:  test 0 '!=' 0
       ../qa/workunits/ceph-helpers.sh:998: wait_for_clean:  sleep 1
       .../qa/workunits/ceph-helpers.sh:999: wait_for_clean:  expr 8 + 1
       ../qa/workunits/ceph-helpers.sh:999: wait_for_clean:  timer=9
       ../qa/workunits/ceph-helpers.sh:985: wait_for_clean:  is_clean
       .../qa/workunits/ceph-helpers.sh:952: is_clean:  get_num_pgs
       .../qa/workunits/ceph-helpers.sh:898: get_num_pgs:  ceph --format xml status
       .../qa/workunits/ceph-helpers.sh:899: get_num_pgs:  xmlstarlet sel -t -m //pgmap/num_pgs -v .
       ../qa/workunits/ceph-helpers.sh:952: is_clean:  num_pgs=4
       ../qa/workunits/ceph-helpers.sh:953: is_clean:  test 4 '!=' 0
       .../qa/workunits/ceph-helpers.sh:954: is_clean:  get_num_active_clean
       .../qa/workunits/ceph-helpers.sh:863: get_num_active_clean:  local 'expression=('
       .../qa/workunits/ceph-helpers.sh:864: get_num_active_clean:  expression+='contains(.,'\''active'\'') and '
       .../qa/workunits/ceph-helpers.sh:865: get_num_active_clean:  expression+='contains(.,'\''clean'\'') and '
       .../qa/workunits/ceph-helpers.sh:866: get_num_active_clean:  expression+='not(contains(.,'\''stale'\''))'
       .../qa/workunits/ceph-helpers.sh:867: get_num_active_clean:  expression+=')'
       .../qa/workunits/ceph-helpers.sh:871: get_num_active_clean:  ceph --format xml pg dump pgs
       .../qa/workunits/ceph-helpers.sh:872: get_num_active_clean:  xmlstarlet sel -t -m '//pg_stat/state[(contains(.,'\''active'\'') and contains(.,'\''clean'\'') and not(contains(.,'\''stale'\'')))]' -v . -n
       .../qa/workunits/ceph-helpers.sh:873: get_num_active_clean:  grep -v '^$'
       .../qa/workunits/ceph-helpers.sh:873: get_num_active_clean:  wc -l
       ../qa/workunits/ceph-helpers.sh:954: is_clean:  test 0 = 4
       ../qa/workunits/ceph-helpers.sh:954: is_clean:  return 1
       ../qa/workunits/ceph-helpers.sh:986: wait_for_clean:  get_is_making_recovery_progress
       .../qa/workunits/ceph-helpers.sh:839: get_is_making_recovery_progress:  xmlstarlet sel -t -m //pgmap/recovering_keys_per_sec -v . -o ' ' -t -m //pgmap/recovering_bytes_per_sec -v . -o ' ' -t -m //pgmap/recovering_objects_per_sec -v .
       .../qa/workunits/ceph-helpers.sh:839: get_is_making_recovery_progress:  ceph --format xml status
       ../qa/workunits/ceph-helpers.sh:839: get_is_making_recovery_progress:  local progress=
       ../qa/workunits/ceph-helpers.sh:840: get_is_making_recovery_progress:  test -n ''
       ../qa/workunits/ceph-helpers.sh:988: wait_for_clean:  ((  timer >= 120  ))
       .../qa/workunits/ceph-helpers.sh:993: wait_for_clean:  get_num_active_clean
       .../qa/workunits/ceph-helpers.sh:863: get_num_active_clean:  local 'expression=('
       .../qa/workunits/ceph-helpers.sh:864: get_num_active_clean:  expression+='contains(.,'\''active'\'') and '
       .../qa/workunits/ceph-helpers.sh:865: get_num_active_clean:  expression+='contains(.,'\''clean'\'') and '
       .../qa/workunits/ceph-helpers.sh:866: get_num_active_clean:  expression+='not(contains(.,'\''stale'\''))'
       .../qa/workunits/ceph-helpers.sh:867: get_num_active_clean:  expression+=')'
       .../qa/workunits/ceph-helpers.sh:871: get_num_active_clean:  ceph --format xml pg dump pgs
       .../qa/workunits/ceph-helpers.sh:872: get_num_active_clean:  xmlstarlet sel -t -m '//pg_stat/state[(contains(.,'\''active'\'') and contains(.,'\''clean'\'') and not(contains(.,'\''stale'\'')))]' -v . -n
       .../qa/workunits/ceph-helpers.sh:873: get_num_active_clean:  grep -v '^$'
       .../qa/workunits/ceph-helpers.sh:873: get_num_active_clean:  wc -l
       ../qa/workunits/ceph-helpers.sh:993: wait_for_clean:  cur_active_clean=0
       ../qa/workunits/ceph-helpers.sh:994: wait_for_clean:  test 0 '!=' 0
       ../qa/workunits/ceph-helpers.sh:998: wait_for_clean:  sleep 1
       .../qa/workunits/ceph-helpers.sh:999: wait_for_clean:  expr 9 + 1
       ../qa/workunits/ceph-helpers.sh:999: wait_for_clean:  timer=10
       ../qa/workunits/ceph-helpers.sh:985: wait_for_clean:  is_clean
       .../qa/workunits/ceph-helpers.sh:952: is_clean:  get_num_pgs
       .../qa/workunits/ceph-helpers.sh:898: get_num_pgs:  ceph --format xml status
       .../qa/workunits/ceph-helpers.sh:899: get_num_pgs:  xmlstarlet sel -t -m //pgmap/num_pgs -v .
       ../qa/workunits/ceph-helpers.sh:952: is_clean:  num_pgs=4
       ../qa/workunits/ceph-helpers.sh:953: is_clean:  test 4 '!=' 0
       .../qa/workunits/ceph-helpers.sh:954: is_clean:  get_num_active_clean
       .../qa/workunits/ceph-helpers.sh:863: get_num_active_clean:  local 'expression=('
       .../qa/workunits/ceph-helpers.sh:864: get_num_active_clean:  expression+='contains(.,'\''active'\'') and '
       .../qa/workunits/ceph-helpers.sh:865: get_num_active_clean:  expression+='contains(.,'\''clean'\'') and '
       .../qa/workunits/ceph-helpers.sh:866: get_num_active_clean:  expression+='not(contains(.,'\''stale'\''))'
       .../qa/workunits/ceph-helpers.sh:867: get_num_active_clean:  expression+=')'
       .../qa/workunits/ceph-helpers.sh:871: get_num_active_clean:  ceph --format xml pg dump pgs
       .../qa/workunits/ceph-helpers.sh:872: get_num_active_clean:  xmlstarlet sel -t -m '//pg_stat/state[(contains(.,'\''active'\'') and contains(.,'\''clean'\'') and not(contains(.,'\''stale'\'')))]' -v . -n
       .../qa/workunits/ceph-helpers.sh:873: get_num_active_clean:  grep -v '^$'
       .../qa/workunits/ceph-helpers.sh:873: get_num_active_clean:  wc -l
       ../qa/workunits/ceph-helpers.sh:954: is_clean:  test 0 = 4
       ../qa/workunits/ceph-helpers.sh:954: is_clean:  return 1
       ../qa/workunits/ceph-helpers.sh:986: wait_for_clean:  get_is_making_recovery_progress
       .../qa/workunits/ceph-helpers.sh:839: get_is_making_recovery_progress:  ceph --format xml status
       .../qa/workunits/ceph-helpers.sh:839: get_is_making_recovery_progress:  xmlstarlet sel -t -m //pgmap/recovering_keys_per_sec -v . -o ' ' -t -m //pgmap/recovering_bytes_per_sec -v . -o ' ' -t -m //pgmap/recovering_objects_per_sec -v .
       ../qa/workunits/ceph-helpers.sh:839: get_is_making_recovery_progress:  local progress=
       ../qa/workunits/ceph-helpers.sh:840: get_is_making_recovery_progress:  test -n ''
       ../qa/workunits/ceph-helpers.sh:988: wait_for_clean:  ((  timer >= 120  ))
       .../qa/workunits/ceph-helpers.sh:993: wait_for_clean:  get_num_active_clean
       .../qa/workunits/ceph-helpers.sh:863: get_num_active_clean:  local 'expression=('
       .../qa/workunits/ceph-helpers.sh:864: get_num_active_clean:  expression+='contains(.,'\''active'\'') and '
       .../qa/workunits/ceph-helpers.sh:865: get_num_active_clean:  expression+='contains(.,'\''clean'\'') and '
       .../qa/workunits/ceph-helpers.sh:866: get_num_active_clean:  expression+='not(contains(.,'\''stale'\''))'
       .../qa/workunits/ceph-helpers.sh:867: get_num_active_clean:  expression+=')'
       .../qa/workunits/ceph-helpers.sh:871: get_num_active_clean:  ceph --format xml pg dump pgs
       .../qa/workunits/ceph-helpers.sh:872: get_num_active_clean:  xmlstarlet sel -t -m '//pg_stat/state[(contains(.,'\''active'\'') and contains(.,'\''clean'\'') and not(contains(.,'\''stale'\'')))]' -v . -n
       .../qa/workunits/ceph-helpers.sh:873: get_num_active_clean:  grep -v '^$'
       .../qa/workunits/ceph-helpers.sh:873: get_num_active_clean:  wc -l
       ../qa/workunits/ceph-helpers.sh:993: wait_for_clean:  cur_active_clean=0
       ../qa/workunits/ceph-helpers.sh:994: wait_for_clean:  test 0 '!=' 0
       ../qa/workunits/ceph-helpers.sh:998: wait_for_clean:  sleep 1
       .../qa/workunits/ceph-helpers.sh:999: wait_for_clean:  expr 10 + 1
       ../qa/workunits/ceph-helpers.sh:999: wait_for_clean:  timer=11
       ../qa/workunits/ceph-helpers.sh:985: wait_for_clean:  is_clean
       .../qa/workunits/ceph-helpers.sh:952: is_clean:  get_num_pgs
       .../qa/workunits/ceph-helpers.sh:898: get_num_pgs:  ceph --format xml status
       .../qa/workunits/ceph-helpers.sh:899: get_num_pgs:  xmlstarlet sel -t -m //pgmap/num_pgs -v .
       ../qa/workunits/ceph-helpers.sh:952: is_clean:  num_pgs=4
       ../qa/workunits/ceph-helpers.sh:953: is_clean:  test 4 '!=' 0
       .../qa/workunits/ceph-helpers.sh:954: is_clean:  get_num_active_clean
       .../qa/workunits/ceph-helpers.sh:863: get_num_active_clean:  local 'expression=('
       .../qa/workunits/ceph-helpers.sh:864: get_num_active_clean:  expression+='contains(.,'\''active'\'') and '
       .../qa/workunits/ceph-helpers.sh:865: get_num_active_clean:  expression+='contains(.,'\''clean'\'') and '
       .../qa/workunits/ceph-helpers.sh:866: get_num_active_clean:  expression+='not(contains(.,'\''stale'\''))'
       .../qa/workunits/ceph-helpers.sh:867: get_num_active_clean:  expression+=')'
       .../qa/workunits/ceph-helpers.sh:871: get_num_active_clean:  ceph --format xml pg dump pgs
       .../qa/workunits/ceph-helpers.sh:872: get_num_active_clean:  xmlstarlet sel -t -m '//pg_stat/state[(contains(.,'\''active'\'') and contains(.,'\''clean'\'') and not(contains(.,'\''stale'\'')))]' -v . -n
       .../qa/workunits/ceph-helpers.sh:873: get_num_active_clean:  grep -v '^$'
       .../qa/workunits/ceph-helpers.sh:873: get_num_active_clean:  wc -l
       ../qa/workunits/ceph-helpers.sh:954: is_clean:  test 0 = 4
       ../qa/workunits/ceph-helpers.sh:954: is_clean:  return 1
       ../qa/workunits/ceph-helpers.sh:986: wait_for_clean:  get_is_making_recovery_progress
       .../qa/workunits/ceph-helpers.sh:839: get_is_making_recovery_progress:  ceph --format xml status
       .../qa/workunits/ceph-helpers.sh:839: get_is_making_recovery_progress:  xmlstarlet sel -t -m //pgmap/recovering_keys_per_sec -v . -o ' ' -t -m //pgmap/recovering_bytes_per_sec -v . -o ' ' -t -m //pgmap/recovering_objects_per_sec -v .
       ../qa/workunits/ceph-helpers.sh:839: get_is_making_recovery_progress:  local progress=
       ../qa/workunits/ceph-helpers.sh:840: get_is_making_recovery_progress:  test -n ''
       ../qa/workunits/ceph-helpers.sh:988: wait_for_clean:  ((  timer >= 120  ))
       .../qa/workunits/ceph-helpers.sh:993: wait_for_clean:  get_num_active_clean
       .../qa/workunits/ceph-helpers.sh:863: get_num_active_clean:  local 'expression=('
       .../qa/workunits/ceph-helpers.sh:864: get_num_active_clean:  expression+='contains(.,'\''active'\'') and '
       .../qa/workunits/ceph-helpers.sh:865: get_num_active_clean:  expression+='contains(.,'\''clean'\'') and '
       .../qa/workunits/ceph-helpers.sh:866: get_num_active_clean:  expression+='not(contains(.,'\''stale'\''))'
       .../qa/workunits/ceph-helpers.sh:867: get_num_active_clean:  expression+=')'
       .../qa/workunits/ceph-helpers.sh:871: get_num_active_clean:  ceph --format xml pg dump pgs
       .../qa/workunits/ceph-helpers.sh:872: get_num_active_clean:  xmlstarlet sel -t -m '//pg_stat/state[(contains(.,'\''active'\'') and contains(.,'\''clean'\'') and not(contains(.,'\''stale'\'')))]' -v . -n
       .../qa/workunits/ceph-helpers.sh:873: get_num_active_clean:  grep -v '^$'
       .../qa/workunits/ceph-helpers.sh:873: get_num_active_clean:  wc -l
       ../qa/workunits/ceph-helpers.sh:993: wait_for_clean:  cur_active_clean=0
       ../qa/workunits/ceph-helpers.sh:994: wait_for_clean:  test 0 '!=' 0
       ../qa/workunits/ceph-helpers.sh:998: wait_for_clean:  sleep 1
       .../qa/workunits/ceph-helpers.sh:999: wait_for_clean:  expr 11 + 1
       ../qa/workunits/ceph-helpers.sh:999: wait_for_clean:  timer=12
       ../qa/workunits/ceph-helpers.sh:985: wait_for_clean:  is_clean
       .../qa/workunits/ceph-helpers.sh:952: is_clean:  get_num_pgs
       .../qa/workunits/ceph-helpers.sh:898: get_num_pgs:  ceph --format xml status
       .../qa/workunits/ceph-helpers.sh:899: get_num_pgs:  xmlstarlet sel -t -m //pgmap/num_pgs -v .
       ../qa/workunits/ceph-helpers.sh:952: is_clean:  num_pgs=4
       ../qa/workunits/ceph-helpers.sh:953: is_clean:  test 4 '!=' 0
       .../qa/workunits/ceph-helpers.sh:954: is_clean:  get_num_active_clean
       .../qa/workunits/ceph-helpers.sh:863: get_num_active_clean:  local 'expression=('
       .../qa/workunits/ceph-helpers.sh:864: get_num_active_clean:  expression+='contains(.,'\''active'\'') and '
       .../qa/workunits/ceph-helpers.sh:865: get_num_active_clean:  expression+='contains(.,'\''clean'\'') and '
       .../qa/workunits/ceph-helpers.sh:866: get_num_active_clean:  expression+='not(contains(.,'\''stale'\''))'
       .../qa/workunits/ceph-helpers.sh:867: get_num_active_clean:  expression+=')'
       .../qa/workunits/ceph-helpers.sh:871: get_num_active_clean:  ceph --format xml pg dump pgs
       .../qa/workunits/ceph-helpers.sh:872: get_num_active_clean:  xmlstarlet sel -t -m '//pg_stat/state[(contains(.,'\''active'\'') and contains(.,'\''clean'\'') and not(contains(.,'\''stale'\'')))]' -v . -n
       .../qa/workunits/ceph-helpers.sh:873: get_num_active_clean:  grep -v '^$'
       .../qa/workunits/ceph-helpers.sh:873: get_num_active_clean:  wc -l
       ../qa/workunits/ceph-helpers.sh:954: is_clean:  test 0 = 4
       ../qa/workunits/ceph-helpers.sh:954: is_clean:  return 1
       ../qa/workunits/ceph-helpers.sh:986: wait_for_clean:  get_is_making_recovery_progress
       .../qa/workunits/ceph-helpers.sh:839: get_is_making_recovery_progress:  ceph --format xml status
       .../qa/workunits/ceph-helpers.sh:839: get_is_making_recovery_progress:  xmlstarlet sel -t -m //pgmap/recovering_keys_per_sec -v . -o ' ' -t -m //pgmap/recovering_bytes_per_sec -v . -o ' ' -t -m //pgmap/recovering_objects_per_sec -v .
       ../qa/workunits/ceph-helpers.sh:839: get_is_making_recovery_progress:  local progress=
       ../qa/workunits/ceph-helpers.sh:840: get_is_making_recovery_progress:  test -n ''
       ../qa/workunits/ceph-helpers.sh:988: wait_for_clean:  ((  timer >= 120  ))
       .../qa/workunits/ceph-helpers.sh:993: wait_for_clean:  get_num_active_clean
       .../qa/workunits/ceph-helpers.sh:863: get_num_active_clean:  local 'expression=('
       .../qa/workunits/ceph-helpers.sh:864: get_num_active_clean:  expression+='contains(.,'\''active'\'') and '
       .../qa/workunits/ceph-helpers.sh:865: get_num_active_clean:  expression+='contains(.,'\''clean'\'') and '
       .../qa/workunits/ceph-helpers.sh:866: get_num_active_clean:  expression+='not(contains(.,'\''stale'\''))'
       .../qa/workunits/ceph-helpers.sh:867: get_num_active_clean:  expression+=')'
       .../qa/workunits/ceph-helpers.sh:871: get_num_active_clean:  ceph --format xml pg dump pgs
       .../qa/workunits/ceph-helpers.sh:872: get_num_active_clean:  xmlstarlet sel -t -m '//pg_stat/state[(contains(.,'\''active'\'') and contains(.,'\''clean'\'') and not(contains(.,'\''stale'\'')))]' -v . -n
       .../qa/workunits/ceph-helpers.sh:873: get_num_active_clean:  grep -v '^$'
       .../qa/workunits/ceph-helpers.sh:873: get_num_active_clean:  wc -l
       ../qa/workunits/ceph-helpers.sh:993: wait_for_clean:  cur_active_clean=0
       ../qa/workunits/ceph-helpers.sh:994: wait_for_clean:  test 0 '!=' 0
       ../qa/workunits/ceph-helpers.sh:998: wait_for_clean:  sleep 1
       .../qa/workunits/ceph-helpers.sh:999: wait_for_clean:  expr 12 + 1
       ../qa/workunits/ceph-helpers.sh:999: wait_for_clean:  timer=13
       ../qa/workunits/ceph-helpers.sh:985: wait_for_clean:  is_clean
       .../qa/workunits/ceph-helpers.sh:952: is_clean:  get_num_pgs
       .../qa/workunits/ceph-helpers.sh:898: get_num_pgs:  ceph --format xml status
       .../qa/workunits/ceph-helpers.sh:899: get_num_pgs:  xmlstarlet sel -t -m //pgmap/num_pgs -v .
       ../qa/workunits/ceph-helpers.sh:952: is_clean:  num_pgs=4
       ../qa/workunits/ceph-helpers.sh:953: is_clean:  test 4 '!=' 0
       .../qa/workunits/ceph-helpers.sh:954: is_clean:  get_num_active_clean
       .../qa/workunits/ceph-helpers.sh:863: get_num_active_clean:  local 'expression=('
       .../qa/workunits/ceph-helpers.sh:864: get_num_active_clean:  expression+='contains(.,'\''active'\'') and '
       .../qa/workunits/ceph-helpers.sh:865: get_num_active_clean:  expression+='contains(.,'\''clean'\'') and '
       .../qa/workunits/ceph-helpers.sh:866: get_num_active_clean:  expression+='not(contains(.,'\''stale'\''))'
       .../qa/workunits/ceph-helpers.sh:867: get_num_active_clean:  expression+=')'
       .../qa/workunits/ceph-helpers.sh:871: get_num_active_clean:  ceph --format xml pg dump pgs
       .../qa/workunits/ceph-helpers.sh:872: get_num_active_clean:  xmlstarlet sel -t -m '//pg_stat/state[(contains(.,'\''active'\'') and contains(.,'\''clean'\'') and not(contains(.,'\''stale'\'')))]' -v . -n
       .../qa/workunits/ceph-helpers.sh:873: get_num_active_clean:  grep -v '^$'
       .../qa/workunits/ceph-helpers.sh:873: get_num_active_clean:  wc -l
       ../qa/workunits/ceph-helpers.sh:954: is_clean:  test 0 = 4
       ../qa/workunits/ceph-helpers.sh:954: is_clean:  return 1
       ../qa/workunits/ceph-helpers.sh:986: wait_for_clean:  get_is_making_recovery_progress
       .../qa/workunits/ceph-helpers.sh:839: get_is_making_recovery_progress:  ceph --format xml status
       .../qa/workunits/ceph-helpers.sh:839: get_is_making_recovery_progress:  xmlstarlet sel -t -m //pgmap/recovering_keys_per_sec -v . -o ' ' -t -m //pgmap/recovering_bytes_per_sec -v . -o ' ' -t -m //pgmap/recovering_objects_per_sec -v .
       ../qa/workunits/ceph-helpers.sh:839: get_is_making_recovery_progress:  local progress=
       ../qa/workunits/ceph-helpers.sh:840: get_is_making_recovery_progress:  test -n ''
       ../qa/workunits/ceph-helpers.sh:988: wait_for_clean:  ((  timer >= 120  ))
       .../qa/workunits/ceph-helpers.sh:993: wait_for_clean:  get_num_active_clean
       .../qa/workunits/ceph-helpers.sh:863: get_num_active_clean:  local 'expression=('
       .../qa/workunits/ceph-helpers.sh:864: get_num_active_clean:  expression+='contains(.,'\''active'\'') and '
       .../qa/workunits/ceph-helpers.sh:865: get_num_active_clean:  expression+='contains(.,'\''clean'\'') and '
       .../qa/workunits/ceph-helpers.sh:866: get_num_active_clean:  expression+='not(contains(.,'\''stale'\''))'
       .../qa/workunits/ceph-helpers.sh:867: get_num_active_clean:  expression+=')'
       .../qa/workunits/ceph-helpers.sh:871: get_num_active_clean:  ceph --format xml pg dump pgs
       .../qa/workunits/ceph-helpers.sh:872: get_num_active_clean:  xmlstarlet sel -t -m '//pg_stat/state[(contains(.,'\''active'\'') and contains(.,'\''clean'\'') and not(contains(.,'\''stale'\'')))]' -v . -n
       .../qa/workunits/ceph-helpers.sh:873: get_num_active_clean:  wc -l
       .../qa/workunits/ceph-helpers.sh:873: get_num_active_clean:  grep -v '^$'
       ../qa/workunits/ceph-helpers.sh:993: wait_for_clean:  cur_active_clean=0
       ../qa/workunits/ceph-helpers.sh:994: wait_for_clean:  test 0 '!=' 0
       ../qa/workunits/ceph-helpers.sh:998: wait_for_clean:  sleep 1
       .../qa/workunits/ceph-helpers.sh:999: wait_for_clean:  expr 13 + 1
       ../qa/workunits/ceph-helpers.sh:999: wait_for_clean:  timer=14
       ../qa/workunits/ceph-helpers.sh:985: wait_for_clean:  is_clean
       .../qa/workunits/ceph-helpers.sh:952: is_clean:  get_num_pgs
       .../qa/workunits/ceph-helpers.sh:898: get_num_pgs:  ceph --format xml status
       .../qa/workunits/ceph-helpers.sh:899: get_num_pgs:  xmlstarlet sel -t -m //pgmap/num_pgs -v .
       ../qa/workunits/ceph-helpers.sh:952: is_clean:  num_pgs=4
       ../qa/workunits/ceph-helpers.sh:953: is_clean:  test 4 '!=' 0
       .../qa/workunits/ceph-helpers.sh:954: is_clean:  get_num_active_clean
       .../qa/workunits/ceph-helpers.sh:863: get_num_active_clean:  local 'expression=('
       .../qa/workunits/ceph-helpers.sh:864: get_num_active_clean:  expression+='contains(.,'\''active'\'') and '
       .../qa/workunits/ceph-helpers.sh:865: get_num_active_clean:  expression+='contains(.,'\''clean'\'') and '
       .../qa/workunits/ceph-helpers.sh:866: get_num_active_clean:  expression+='not(contains(.,'\''stale'\''))'
       .../qa/workunits/ceph-helpers.sh:867: get_num_active_clean:  expression+=')'
       .../qa/workunits/ceph-helpers.sh:871: get_num_active_clean:  ceph --format xml pg dump pgs
       .../qa/workunits/ceph-helpers.sh:872: get_num_active_clean:  xmlstarlet sel -t -m '//pg_stat/state[(contains(.,'\''active'\'') and contains(.,'\''clean'\'') and not(contains(.,'\''stale'\'')))]' -v . -n
       .../qa/workunits/ceph-helpers.sh:873: get_num_active_clean:  grep -v '^$'
       .../qa/workunits/ceph-helpers.sh:873: get_num_active_clean:  wc -l
       ../qa/workunits/ceph-helpers.sh:954: is_clean:  test 0 = 4
       ../qa/workunits/ceph-helpers.sh:954: is_clean:  return 1
       ../qa/workunits/ceph-helpers.sh:986: wait_for_clean:  get_is_making_recovery_progress
       .../qa/workunits/ceph-helpers.sh:839: get_is_making_recovery_progress:  ceph --format xml status
       .../qa/workunits/ceph-helpers.sh:839: get_is_making_recovery_progress:  xmlstarlet sel -t -m //pgmap/recovering_keys_per_sec -v . -o ' ' -t -m //pgmap/recovering_bytes_per_sec -v . -o ' ' -t -m //pgmap/recovering_objects_per_sec -v .
       ../qa/workunits/ceph-helpers.sh:839: get_is_making_recovery_progress:  local progress=
       ../qa/workunits/ceph-helpers.sh:840: get_is_making_recovery_progress:  test -n ''
       ../qa/workunits/ceph-helpers.sh:988: wait_for_clean:  ((  timer >= 120  ))
       .../qa/workunits/ceph-helpers.sh:993: wait_for_clean:  get_num_active_clean
       .../qa/workunits/ceph-helpers.sh:863: get_num_active_clean:  local 'expression=('
       .../qa/workunits/ceph-helpers.sh:864: get_num_active_clean:  expression+='contains(.,'\''active'\'') and '
       .../qa/workunits/ceph-helpers.sh:865: get_num_active_clean:  expression+='contains(.,'\''clean'\'') and '
       .../qa/workunits/ceph-helpers.sh:866: get_num_active_clean:  expression+='not(contains(.,'\''stale'\''))'
       .../qa/workunits/ceph-helpers.sh:867: get_num_active_clean:  expression+=')'
       .../qa/workunits/ceph-helpers.sh:871: get_num_active_clean:  ceph --format xml pg dump pgs
       .../qa/workunits/ceph-helpers.sh:873: get_num_active_clean:  wc -l
       .../qa/workunits/ceph-helpers.sh:872: get_num_active_clean:  xmlstarlet sel -t -m '//pg_stat/state[(contains(.,'\''active'\'') and contains(.,'\''clean'\'') and not(contains(.,'\''stale'\'')))]' -v . -n
       .../qa/workunits/ceph-helpers.sh:873: get_num_active_clean:  grep -v '^$'
       ../qa/workunits/ceph-helpers.sh:993: wait_for_clean:  cur_active_clean=0
       ../qa/workunits/ceph-helpers.sh:994: wait_for_clean:  test 0 '!=' 0
       ../qa/workunits/ceph-helpers.sh:998: wait_for_clean:  sleep 1
       .../qa/workunits/ceph-helpers.sh:999: wait_for_clean:  expr 14 + 1
       ../qa/workunits/ceph-helpers.sh:999: wait_for_clean:  timer=15
       ../qa/workunits/ceph-helpers.sh:985: wait_for_clean:  is_clean
       .../qa/workunits/ceph-helpers.sh:952: is_clean:  get_num_pgs
       .../qa/workunits/ceph-helpers.sh:898: get_num_pgs:  ceph --format xml status
       .../qa/workunits/ceph-helpers.sh:899: get_num_pgs:  xmlstarlet sel -t -m //pgmap/num_pgs -v .
       ../qa/workunits/ceph-helpers.sh:952: is_clean:  num_pgs=4
       ../qa/workunits/ceph-helpers.sh:953: is_clean:  test 4 '!=' 0
       .../qa/workunits/ceph-helpers.sh:954: is_clean:  get_num_active_clean
       .../qa/workunits/ceph-helpers.sh:863: get_num_active_clean:  local 'expression=('
       .../qa/workunits/ceph-helpers.sh:864: get_num_active_clean:  expression+='contains(.,'\''active'\'') and '
       .../qa/workunits/ceph-helpers.sh:865: get_num_active_clean:  expression+='contains(.,'\''clean'\'') and '
       .../qa/workunits/ceph-helpers.sh:866: get_num_active_clean:  expression+='not(contains(.,'\''stale'\''))'
       .../qa/workunits/ceph-helpers.sh:867: get_num_active_clean:  expression+=')'
       .../qa/workunits/ceph-helpers.sh:871: get_num_active_clean:  ceph --format xml pg dump pgs
       .../qa/workunits/ceph-helpers.sh:872: get_num_active_clean:  xmlstarlet sel -t -m '//pg_stat/state[(contains(.,'\''active'\'') and contains(.,'\''clean'\'') and not(contains(.,'\''stale'\'')))]' -v . -n
       .../qa/workunits/ceph-helpers.sh:873: get_num_active_clean:  grep -v '^$'
       .../qa/workunits/ceph-helpers.sh:873: get_num_active_clean:  wc -l
       ../qa/workunits/ceph-helpers.sh:954: is_clean:  test 0 = 4
       ../qa/workunits/ceph-helpers.sh:954: is_clean:  return 1
       ../qa/workunits/ceph-helpers.sh:986: wait_for_clean:  get_is_making_recovery_progress
       .../qa/workunits/ceph-helpers.sh:839: get_is_making_recovery_progress:  ceph --format xml status
       .../qa/workunits/ceph-helpers.sh:839: get_is_making_recovery_progress:  xmlstarlet sel -t -m //pgmap/recovering_keys_per_sec -v . -o ' ' -t -m //pgmap/recovering_bytes_per_sec -v . -o ' ' -t -m //pgmap/recovering_objects_per_sec -v .
       ../qa/workunits/ceph-helpers.sh:839: get_is_making_recovery_progress:  local progress=
       ../qa/workunits/ceph-helpers.sh:840: get_is_making_recovery_progress:  test -n ''
       ../qa/workunits/ceph-helpers.sh:988: wait_for_clean:  ((  timer >= 120  ))
       .../qa/workunits/ceph-helpers.sh:993: wait_for_clean:  get_num_active_clean
       .../qa/workunits/ceph-helpers.sh:863: get_num_active_clean:  local 'expression=('
       .../qa/workunits/ceph-helpers.sh:864: get_num_active_clean:  expression+='contains(.,'\''active'\'') and '
       .../qa/workunits/ceph-helpers.sh:865: get_num_active_clean:  expression+='contains(.,'\''clean'\'') and '
       .../qa/workunits/ceph-helpers.sh:866: get_num_active_clean:  expression+='not(contains(.,'\''stale'\''))'
       .../qa/workunits/ceph-helpers.sh:867: get_num_active_clean:  expression+=')'
       .../qa/workunits/ceph-helpers.sh:871: get_num_active_clean:  ceph --format xml pg dump pgs
       .../qa/workunits/ceph-helpers.sh:872: get_num_active_clean:  xmlstarlet sel -t -m '//pg_stat/state[(contains(.,'\''active'\'') and contains(.,'\''clean'\'') and not(contains(.,'\''stale'\'')))]' -v . -n
       .../qa/workunits/ceph-helpers.sh:873: get_num_active_clean:  grep -v '^$'
       .../qa/workunits/ceph-helpers.sh:873: get_num_active_clean:  wc -l
       ../qa/workunits/ceph-helpers.sh:993: wait_for_clean:  cur_active_clean=0
       ../qa/workunits/ceph-helpers.sh:994: wait_for_clean:  test 0 '!=' 0
       ../qa/workunits/ceph-helpers.sh:998: wait_for_clean:  sleep 1
       .../qa/workunits/ceph-helpers.sh:999: wait_for_clean:  expr 15 + 1
       ../qa/workunits/ceph-helpers.sh:999: wait_for_clean:  timer=16
       ../qa/workunits/ceph-helpers.sh:985: wait_for_clean:  is_clean
       .../qa/workunits/ceph-helpers.sh:952: is_clean:  get_num_pgs
       .../qa/workunits/ceph-helpers.sh:898: get_num_pgs:  ceph --format xml status
       .../qa/workunits/ceph-helpers.sh:899: get_num_pgs:  xmlstarlet sel -t -m //pgmap/num_pgs -v .
       ../qa/workunits/ceph-helpers.sh:952: is_clean:  num_pgs=4
       ../qa/workunits/ceph-helpers.sh:953: is_clean:  test 4 '!=' 0
       .../qa/workunits/ceph-helpers.sh:954: is_clean:  get_num_active_clean
       .../qa/workunits/ceph-helpers.sh:863: get_num_active_clean:  local 'expression=('
       .../qa/workunits/ceph-helpers.sh:864: get_num_active_clean:  expression+='contains(.,'\''active'\'') and '
       .../qa/workunits/ceph-helpers.sh:865: get_num_active_clean:  expression+='contains(.,'\''clean'\'') and '
       .../qa/workunits/ceph-helpers.sh:866: get_num_active_clean:  expression+='not(contains(.,'\''stale'\''))'
       .../qa/workunits/ceph-helpers.sh:867: get_num_active_clean:  expression+=')'
       .../qa/workunits/ceph-helpers.sh:871: get_num_active_clean:  ceph --format xml pg dump pgs
       .../qa/workunits/ceph-helpers.sh:872: get_num_active_clean:  xmlstarlet sel -t -m '//pg_stat/state[(contains(.,'\''active'\'') and contains(.,'\''clean'\'') and not(contains(.,'\''stale'\'')))]' -v . -n
       .../qa/workunits/ceph-helpers.sh:873: get_num_active_clean:  grep -v '^$'
       .../qa/workunits/ceph-helpers.sh:873: get_num_active_clean:  wc -l
       ../qa/workunits/ceph-helpers.sh:954: is_clean:  test 0 = 4
       ../qa/workunits/ceph-helpers.sh:954: is_clean:  return 1
       ../qa/workunits/ceph-helpers.sh:986: wait_for_clean:  get_is_making_recovery_progress
       .../qa/workunits/ceph-helpers.sh:839: get_is_making_recovery_progress:  ceph --format xml status
       .../qa/workunits/ceph-helpers.sh:839: get_is_making_recovery_progress:  xmlstarlet sel -t -m //pgmap/recovering_keys_per_sec -v . -o ' ' -t -m //pgmap/recovering_bytes_per_sec -v . -o ' ' -t -m //pgmap/recovering_objects_per_sec -v .
       ../qa/workunits/ceph-helpers.sh:839: get_is_making_recovery_progress:  local progress=
       ../qa/workunits/ceph-helpers.sh:840: get_is_making_recovery_progress:  test -n ''
       ../qa/workunits/ceph-helpers.sh:988: wait_for_clean:  ((  timer >= 120  ))
       .../qa/workunits/ceph-helpers.sh:993: wait_for_clean:  get_num_active_clean
       .../qa/workunits/ceph-helpers.sh:863: get_num_active_clean:  local 'expression=('
       .../qa/workunits/ceph-helpers.sh:864: get_num_active_clean:  expression+='contains(.,'\''active'\'') and '
       .../qa/workunits/ceph-helpers.sh:865: get_num_active_clean:  expression+='contains(.,'\''clean'\'') and '
       .../qa/workunits/ceph-helpers.sh:866: get_num_active_clean:  expression+='not(contains(.,'\''stale'\''))'
       .../qa/workunits/ceph-helpers.sh:867: get_num_active_clean:  expression+=')'
       .../qa/workunits/ceph-helpers.sh:871: get_num_active_clean:  ceph --format xml pg dump pgs
       .../qa/workunits/ceph-helpers.sh:873: get_num_active_clean:  grep -v '^$'
       .../qa/workunits/ceph-helpers.sh:872: get_num_active_clean:  xmlstarlet sel -t -m '//pg_stat/state[(contains(.,'\''active'\'') and contains(.,'\''clean'\'') and not(contains(.,'\''stale'\'')))]' -v . -n
       .../qa/workunits/ceph-helpers.sh:873: get_num_active_clean:  wc -l
       ../qa/workunits/ceph-helpers.sh:993: wait_for_clean:  cur_active_clean=0
       ../qa/workunits/ceph-helpers.sh:994: wait_for_clean:  test 0 '!=' 0
       ../qa/workunits/ceph-helpers.sh:998: wait_for_clean:  sleep 1
       .../qa/workunits/ceph-helpers.sh:999: wait_for_clean:  expr 16 + 1
       ../qa/workunits/ceph-helpers.sh:999: wait_for_clean:  timer=17
       ../qa/workunits/ceph-helpers.sh:985: wait_for_clean:  is_clean
       .../qa/workunits/ceph-helpers.sh:952: is_clean:  get_num_pgs
       .../qa/workunits/ceph-helpers.sh:898: get_num_pgs:  ceph --format xml status
       .../qa/workunits/ceph-helpers.sh:899: get_num_pgs:  xmlstarlet sel -t -m //pgmap/num_pgs -v .
       ../qa/workunits/ceph-helpers.sh:952: is_clean:  num_pgs=4
       ../qa/workunits/ceph-helpers.sh:953: is_clean:  test 4 '!=' 0
       .../qa/workunits/ceph-helpers.sh:954: is_clean:  get_num_active_clean
       .../qa/workunits/ceph-helpers.sh:863: get_num_active_clean:  local 'expression=('
       .../qa/workunits/ceph-helpers.sh:864: get_num_active_clean:  expression+='contains(.,'\''active'\'') and '
       .../qa/workunits/ceph-helpers.sh:865: get_num_active_clean:  expression+='contains(.,'\''clean'\'') and '
       .../qa/workunits/ceph-helpers.sh:866: get_num_active_clean:  expression+='not(contains(.,'\''stale'\''))'
       .../qa/workunits/ceph-helpers.sh:867: get_num_active_clean:  expression+=')'
       .../qa/workunits/ceph-helpers.sh:871: get_num_active_clean:  ceph --format xml pg dump pgs
       .../qa/workunits/ceph-helpers.sh:872: get_num_active_clean:  xmlstarlet sel -t -m '//pg_stat/state[(contains(.,'\''active'\'') and contains(.,'\''clean'\'') and not(contains(.,'\''stale'\'')))]' -v . -n
       .../qa/workunits/ceph-helpers.sh:873: get_num_active_clean:  grep -v '^$'
       .../qa/workunits/ceph-helpers.sh:873: get_num_active_clean:  wc -l
       ../qa/workunits/ceph-helpers.sh:954: is_clean:  test 0 = 4
       ../qa/workunits/ceph-helpers.sh:954: is_clean:  return 1
       ../qa/workunits/ceph-helpers.sh:986: wait_for_clean:  get_is_making_recovery_progress
       .../qa/workunits/ceph-helpers.sh:839: get_is_making_recovery_progress:  xmlstarlet sel -t -m //pgmap/recovering_keys_per_sec -v . -o ' ' -t -m //pgmap/recovering_bytes_per_sec -v . -o ' ' -t -m //pgmap/recovering_objects_per_sec -v .
       .../qa/workunits/ceph-helpers.sh:839: get_is_making_recovery_progress:  ceph --format xml status
       ../qa/workunits/ceph-helpers.sh:839: get_is_making_recovery_progress:  local progress=
       ../qa/workunits/ceph-helpers.sh:840: get_is_making_recovery_progress:  test -n ''
       ../qa/workunits/ceph-helpers.sh:988: wait_for_clean:  ((  timer >= 120  ))
       .../qa/workunits/ceph-helpers.sh:993: wait_for_clean:  get_num_active_clean
       .../qa/workunits/ceph-helpers.sh:863: get_num_active_clean:  local 'expression=('
       .../qa/workunits/ceph-helpers.sh:864: get_num_active_clean:  expression+='contains(.,'\''active'\'') and '
       .../qa/workunits/ceph-helpers.sh:865: get_num_active_clean:  expression+='contains(.,'\''clean'\'') and '
       .../qa/workunits/ceph-helpers.sh:866: get_num_active_clean:  expression+='not(contains(.,'\''stale'\''))'
       .../qa/workunits/ceph-helpers.sh:867: get_num_active_clean:  expression+=')'
       .../qa/workunits/ceph-helpers.sh:873: get_num_active_clean:  grep -v '^$'
       .../qa/workunits/ceph-helpers.sh:871: get_num_active_clean:  ceph --format xml pg dump pgs
       .../qa/workunits/ceph-helpers.sh:873: get_num_active_clean:  wc -l
       .../qa/workunits/ceph-helpers.sh:872: get_num_active_clean:  xmlstarlet sel -t -m '//pg_stat/state[(contains(.,'\''active'\'') and contains(.,'\''clean'\'') and not(contains(.,'\''stale'\'')))]' -v . -n
       ../qa/workunits/ceph-helpers.sh:993: wait_for_clean:  cur_active_clean=0
       ../qa/workunits/ceph-helpers.sh:994: wait_for_clean:  test 0 '!=' 0
       ../qa/workunits/ceph-helpers.sh:998: wait_for_clean:  sleep 1
       .../qa/workunits/ceph-helpers.sh:999: wait_for_clean:  expr 17 + 1
       ../qa/workunits/ceph-helpers.sh:999: wait_for_clean:  timer=18
       ../qa/workunits/ceph-helpers.sh:985: wait_for_clean:  is_clean
       .../qa/workunits/ceph-helpers.sh:952: is_clean:  get_num_pgs
       .../qa/workunits/ceph-helpers.sh:899: get_num_pgs:  xmlstarlet sel -t -m //pgmap/num_pgs -v .
       .../qa/workunits/ceph-helpers.sh:898: get_num_pgs:  ceph --format xml status
       ../qa/workunits/ceph-helpers.sh:952: is_clean:  num_pgs=4
       ../qa/workunits/ceph-helpers.sh:953: is_clean:  test 4 '!=' 0
       .../qa/workunits/ceph-helpers.sh:954: is_clean:  get_num_active_clean
       .../qa/workunits/ceph-helpers.sh:863: get_num_active_clean:  local 'expression=('
       .../qa/workunits/ceph-helpers.sh:864: get_num_active_clean:  expression+='contains(.,'\''active'\'') and '
       .../qa/workunits/ceph-helpers.sh:865: get_num_active_clean:  expression+='contains(.,'\''clean'\'') and '
       .../qa/workunits/ceph-helpers.sh:866: get_num_active_clean:  expression+='not(contains(.,'\''stale'\''))'
       .../qa/workunits/ceph-helpers.sh:867: get_num_active_clean:  expression+=')'
       .../qa/workunits/ceph-helpers.sh:871: get_num_active_clean:  ceph --format xml pg dump pgs
       .../qa/workunits/ceph-helpers.sh:872: get_num_active_clean:  xmlstarlet sel -t -m '//pg_stat/state[(contains(.,'\''active'\'') and contains(.,'\''clean'\'') and not(contains(.,'\''stale'\'')))]' -v . -n
       .../qa/workunits/ceph-helpers.sh:873: get_num_active_clean:  grep -v '^$'
       .../qa/workunits/ceph-helpers.sh:873: get_num_active_clean:  wc -l
       ../qa/workunits/ceph-helpers.sh:954: is_clean:  test 0 = 4
       ../qa/workunits/ceph-helpers.sh:954: is_clean:  return 1
       ../qa/workunits/ceph-helpers.sh:986: wait_for_clean:  get_is_making_recovery_progress
       .../qa/workunits/ceph-helpers.sh:839: get_is_making_recovery_progress:  ceph --format xml status
       .../qa/workunits/ceph-helpers.sh:839: get_is_making_recovery_progress:  xmlstarlet sel -t -m //pgmap/recovering_keys_per_sec -v . -o ' ' -t -m //pgmap/recovering_bytes_per_sec -v . -o ' ' -t -m //pgmap/recovering_objects_per_sec -v .
       ../qa/workunits/ceph-helpers.sh:839: get_is_making_recovery_progress:  local progress=
       ../qa/workunits/ceph-helpers.sh:840: get_is_making_recovery_progress:  test -n ''
       ../qa/workunits/ceph-helpers.sh:988: wait_for_clean:  ((  timer >= 120  ))
       .../qa/workunits/ceph-helpers.sh:993: wait_for_clean:  get_num_active_clean
       .../qa/workunits/ceph-helpers.sh:863: get_num_active_clean:  local 'expression=('
       .../qa/workunits/ceph-helpers.sh:864: get_num_active_clean:  expression+='contains(.,'\''active'\'') and '
       .../qa/workunits/ceph-helpers.sh:865: get_num_active_clean:  expression+='contains(.,'\''clean'\'') and '
       .../qa/workunits/ceph-helpers.sh:866: get_num_active_clean:  expression+='not(contains(.,'\''stale'\''))'
       .../qa/workunits/ceph-helpers.sh:867: get_num_active_clean:  expression+=')'
       .../qa/workunits/ceph-helpers.sh:871: get_num_active_clean:  ceph --format xml pg dump pgs
       .../qa/workunits/ceph-helpers.sh:872: get_num_active_clean:  xmlstarlet sel -t -m '//pg_stat/state[(contains(.,'\''active'\'') and contains(.,'\''clean'\'') and not(contains(.,'\''stale'\'')))]' -v . -n
       .../qa/workunits/ceph-helpers.sh:873: get_num_active_clean:  wc -l
       .../qa/workunits/ceph-helpers.sh:873: get_num_active_clean:  grep -v '^$'
       ../qa/workunits/ceph-helpers.sh:993: wait_for_clean:  cur_active_clean=0
       ../qa/workunits/ceph-helpers.sh:994: wait_for_clean:  test 0 '!=' 0
       ../qa/workunits/ceph-helpers.sh:998: wait_for_clean:  sleep 1
       .../qa/workunits/ceph-helpers.sh:999: wait_for_clean:  expr 18 + 1
       ../qa/workunits/ceph-helpers.sh:999: wait_for_clean:  timer=19
       ../qa/workunits/ceph-helpers.sh:985: wait_for_clean:  is_clean
       .../qa/workunits/ceph-helpers.sh:952: is_clean:  get_num_pgs
       .../qa/workunits/ceph-helpers.sh:898: get_num_pgs:  ceph --format xml status
       .../qa/workunits/ceph-helpers.sh:899: get_num_pgs:  xmlstarlet sel -t -m //pgmap/num_pgs -v .
       ../qa/workunits/ceph-helpers.sh:952: is_clean:  num_pgs=4
       ../qa/workunits/ceph-helpers.sh:953: is_clean:  test 4 '!=' 0
       .../qa/workunits/ceph-helpers.sh:954: is_clean:  get_num_active_clean
       .../qa/workunits/ceph-helpers.sh:863: get_num_active_clean:  local 'expression=('
       .../qa/workunits/ceph-helpers.sh:864: get_num_active_clean:  expression+='contains(.,'\''active'\'') and '
       .../qa/workunits/ceph-helpers.sh:865: get_num_active_clean:  expression+='contains(.,'\''clean'\'') and '
       .../qa/workunits/ceph-helpers.sh:866: get_num_active_clean:  expression+='not(contains(.,'\''stale'\''))'
       .../qa/workunits/ceph-helpers.sh:867: get_num_active_clean:  expression+=')'
       .../qa/workunits/ceph-helpers.sh:871: get_num_active_clean:  ceph --format xml pg dump pgs
       .../qa/workunits/ceph-helpers.sh:872: get_num_active_clean:  xmlstarlet sel -t -m '//pg_stat/state[(contains(.,'\''active'\'') and contains(.,'\''clean'\'') and not(contains(.,'\''stale'\'')))]' -v . -n
       .../qa/workunits/ceph-helpers.sh:873: get_num_active_clean:  grep -v '^$'
       .../qa/workunits/ceph-helpers.sh:873: get_num_active_clean:  wc -l
       ../qa/workunits/ceph-helpers.sh:954: is_clean:  test 0 = 4
       ../qa/workunits/ceph-helpers.sh:954: is_clean:  return 1
       ../qa/workunits/ceph-helpers.sh:986: wait_for_clean:  get_is_making_recovery_progress
       .../qa/workunits/ceph-helpers.sh:839: get_is_making_recovery_progress:  ceph --format xml status
       .../qa/workunits/ceph-helpers.sh:839: get_is_making_recovery_progress:  xmlstarlet sel -t -m //pgmap/recovering_keys_per_sec -v . -o ' ' -t -m //pgmap/recovering_bytes_per_sec -v . -o ' ' -t -m //pgmap/recovering_objects_per_sec -v .
       ../qa/workunits/ceph-helpers.sh:839: get_is_making_recovery_progress:  local progress=
       ../qa/workunits/ceph-helpers.sh:840: get_is_making_recovery_progress:  test -n ''
       ../qa/workunits/ceph-helpers.sh:988: wait_for_clean:  ((  timer >= 120  ))
       .../qa/workunits/ceph-helpers.sh:993: wait_for_clean:  get_num_active_clean
       .../qa/workunits/ceph-helpers.sh:863: get_num_active_clean:  local 'expression=('
       .../qa/workunits/ceph-helpers.sh:864: get_num_active_clean:  expression+='contains(.,'\''active'\'') and '
       .../qa/workunits/ceph-helpers.sh:865: get_num_active_clean:  expression+='contains(.,'\''clean'\'') and '
       .../qa/workunits/ceph-helpers.sh:866: get_num_active_clean:  expression+='not(contains(.,'\''stale'\''))'
       .../qa/workunits/ceph-helpers.sh:867: get_num_active_clean:  expression+=')'
       .../qa/workunits/ceph-helpers.sh:871: get_num_active_clean:  ceph --format xml pg dump pgs
       .../qa/workunits/ceph-helpers.sh:872: get_num_active_clean:  xmlstarlet sel -t -m '//pg_stat/state[(contains(.,'\''active'\'') and contains(.,'\''clean'\'') and not(contains(.,'\''stale'\'')))]' -v . -n
       .../qa/workunits/ceph-helpers.sh:873: get_num_active_clean:  grep -v '^$'
       .../qa/workunits/ceph-helpers.sh:873: get_num_active_clean:  wc -l
       ../qa/workunits/ceph-helpers.sh:993: wait_for_clean:  cur_active_clean=0
       ../qa/workunits/ceph-helpers.sh:994: wait_for_clean:  test 0 '!=' 0
       ../qa/workunits/ceph-helpers.sh:998: wait_for_clean:  sleep 1
       .../qa/workunits/ceph-helpers.sh:999: wait_for_clean:  expr 19 + 1
       ../qa/workunits/ceph-helpers.sh:999: wait_for_clean:  timer=20
       ../qa/workunits/ceph-helpers.sh:985: wait_for_clean:  is_clean
       .../qa/workunits/ceph-helpers.sh:952: is_clean:  get_num_pgs
       .../qa/workunits/ceph-helpers.sh:898: get_num_pgs:  ceph --format xml status
       .../qa/workunits/ceph-helpers.sh:899: get_num_pgs:  xmlstarlet sel -t -m //pgmap/num_pgs -v .
       ../qa/workunits/ceph-helpers.sh:952: is_clean:  num_pgs=4
       ../qa/workunits/ceph-helpers.sh:953: is_clean:  test 4 '!=' 0
       .../qa/workunits/ceph-helpers.sh:954: is_clean:  get_num_active_clean
       .../qa/workunits/ceph-helpers.sh:863: get_num_active_clean:  local 'expression=('
       .../qa/workunits/ceph-helpers.sh:864: get_num_active_clean:  expression+='contains(.,'\''active'\'') and '
       .../qa/workunits/ceph-helpers.sh:865: get_num_active_clean:  expression+='contains(.,'\''clean'\'') and '
       .../qa/workunits/ceph-helpers.sh:866: get_num_active_clean:  expression+='not(contains(.,'\''stale'\''))'
       .../qa/workunits/ceph-helpers.sh:867: get_num_active_clean:  expression+=')'
       .../qa/workunits/ceph-helpers.sh:871: get_num_active_clean:  ceph --format xml pg dump pgs
       .../qa/workunits/ceph-helpers.sh:872: get_num_active_clean:  xmlstarlet sel -t -m '//pg_stat/state[(contains(.,'\''active'\'') and contains(.,'\''clean'\'') and not(contains(.,'\''stale'\'')))]' -v . -n
       .../qa/workunits/ceph-helpers.sh:873: get_num_active_clean:  wc -l
       .../qa/workunits/ceph-helpers.sh:873: get_num_active_clean:  grep -v '^$'
       ../qa/workunits/ceph-helpers.sh:954: is_clean:  test 0 = 4
       ../qa/workunits/ceph-helpers.sh:954: is_clean:  return 1
       ../qa/workunits/ceph-helpers.sh:986: wait_for_clean:  get_is_making_recovery_progress
       .../qa/workunits/ceph-helpers.sh:839: get_is_making_recovery_progress:  ceph --format xml status
       .../qa/workunits/ceph-helpers.sh:839: get_is_making_recovery_progress:  xmlstarlet sel -t -m //pgmap/recovering_keys_per_sec -v . -o ' ' -t -m //pgmap/recovering_bytes_per_sec -v . -o ' ' -t -m //pgmap/recovering_objects_per_sec -v .
       ../qa/workunits/ceph-helpers.sh:839: get_is_making_recovery_progress:  local progress=
       ../qa/workunits/ceph-helpers.sh:840: get_is_making_recovery_progress:  test -n ''
       ../qa/workunits/ceph-helpers.sh:988: wait_for_clean:  ((  timer >= 120  ))
       .../qa/workunits/ceph-helpers.sh:993: wait_for_clean:  get_num_active_clean
       .../qa/workunits/ceph-helpers.sh:863: get_num_active_clean:  local 'expression=('
       .../qa/workunits/ceph-helpers.sh:864: get_num_active_clean:  expression+='contains(.,'\''active'\'') and '
       .../qa/workunits/ceph-helpers.sh:865: get_num_active_clean:  expression+='contains(.,'\''clean'\'') and '
       .../qa/workunits/ceph-helpers.sh:866: get_num_active_clean:  expression+='not(contains(.,'\''stale'\''))'
       .../qa/workunits/ceph-helpers.sh:867: get_num_active_clean:  expression+=')'
       .../qa/workunits/ceph-helpers.sh:873: get_num_active_clean:  wc -l
       .../qa/workunits/ceph-helpers.sh:873: get_num_active_clean:  grep -v '^$'
       .../qa/workunits/ceph-helpers.sh:872: get_num_active_clean:  xmlstarlet sel -t -m '//pg_stat/state[(contains(.,'\''active'\'') and contains(.,'\''clean'\'') and not(contains(.,'\''stale'\'')))]' -v . -n
       .../qa/workunits/ceph-helpers.sh:871: get_num_active_clean:  ceph --format xml pg dump pgs
       ../qa/workunits/ceph-helpers.sh:993: wait_for_clean:  cur_active_clean=0
       ../qa/workunits/ceph-helpers.sh:994: wait_for_clean:  test 0 '!=' 0
       ../qa/workunits/ceph-helpers.sh:998: wait_for_clean:  sleep 1
       .../qa/workunits/ceph-helpers.sh:999: wait_for_clean:  expr 20 + 1
       ../qa/workunits/ceph-helpers.sh:999: wait_for_clean:  timer=21
       ../qa/workunits/ceph-helpers.sh:985: wait_for_clean:  is_clean
       .../qa/workunits/ceph-helpers.sh:952: is_clean:  get_num_pgs
       .../qa/workunits/ceph-helpers.sh:898: get_num_pgs:  ceph --format xml status
       .../qa/workunits/ceph-helpers.sh:899: get_num_pgs:  xmlstarlet sel -t -m //pgmap/num_pgs -v .
       ../qa/workunits/ceph-helpers.sh:952: is_clean:  num_pgs=4
       ../qa/workunits/ceph-helpers.sh:953: is_clean:  test 4 '!=' 0
       .../qa/workunits/ceph-helpers.sh:954: is_clean:  get_num_active_clean
       .../qa/workunits/ceph-helpers.sh:863: get_num_active_clean:  local 'expression=('
       .../qa/workunits/ceph-helpers.sh:864: get_num_active_clean:  expression+='contains(.,'\''active'\'') and '
       .../qa/workunits/ceph-helpers.sh:865: get_num_active_clean:  expression+='contains(.,'\''clean'\'') and '
       .../qa/workunits/ceph-helpers.sh:866: get_num_active_clean:  expression+='not(contains(.,'\''stale'\''))'
       .../qa/workunits/ceph-helpers.sh:867: get_num_active_clean:  expression+=')'
       .../qa/workunits/ceph-helpers.sh:871: get_num_active_clean:  ceph --format xml pg dump pgs
       .../qa/workunits/ceph-helpers.sh:872: get_num_active_clean:  xmlstarlet sel -t -m '//pg_stat/state[(contains(.,'\''active'\'') and contains(.,'\''clean'\'') and not(contains(.,'\''stale'\'')))]' -v . -n
       .../qa/workunits/ceph-helpers.sh:873: get_num_active_clean:  wc -l
       .../qa/workunits/ceph-helpers.sh:873: get_num_active_clean:  grep -v '^$'
       ../qa/workunits/ceph-helpers.sh:954: is_clean:  test 0 = 4
       ../qa/workunits/ceph-helpers.sh:954: is_clean:  return 1
       ../qa/workunits/ceph-helpers.sh:986: wait_for_clean:  get_is_making_recovery_progress
       .../qa/workunits/ceph-helpers.sh:839: get_is_making_recovery_progress:  ceph --format xml status
       .../qa/workunits/ceph-helpers.sh:839: get_is_making_recovery_progress:  xmlstarlet sel -t -m //pgmap/recovering_keys_per_sec -v . -o ' ' -t -m //pgmap/recovering_bytes_per_sec -v . -o ' ' -t -m //pgmap/recovering_objects_per_sec -v .
       ../qa/workunits/ceph-helpers.sh:839: get_is_making_recovery_progress:  local progress=
       ../qa/workunits/ceph-helpers.sh:840: get_is_making_recovery_progress:  test -n ''
       ../qa/workunits/ceph-helpers.sh:988: wait_for_clean:  ((  timer >= 120  ))
       .../qa/workunits/ceph-helpers.sh:993: wait_for_clean:  get_num_active_clean
       .../qa/workunits/ceph-helpers.sh:863: get_num_active_clean:  local 'expression=('
       .../qa/workunits/ceph-helpers.sh:864: get_num_active_clean:  expression+='contains(.,'\''active'\'') and '
       .../qa/workunits/ceph-helpers.sh:865: get_num_active_clean:  expression+='contains(.,'\''clean'\'') and '
       .../qa/workunits/ceph-helpers.sh:866: get_num_active_clean:  expression+='not(contains(.,'\''stale'\''))'
       .../qa/workunits/ceph-helpers.sh:867: get_num_active_clean:  expression+=')'
       .../qa/workunits/ceph-helpers.sh:871: get_num_active_clean:  ceph --format xml pg dump pgs
       .../qa/workunits/ceph-helpers.sh:872: get_num_active_clean:  xmlstarlet sel -t -m '//pg_stat/state[(contains(.,'\''active'\'') and contains(.,'\''clean'\'') and not(contains(.,'\''stale'\'')))]' -v . -n
       .../qa/workunits/ceph-helpers.sh:873: get_num_active_clean:  wc -l
       .../qa/workunits/ceph-helpers.sh:873: get_num_active_clean:  grep -v '^$'
       ../qa/workunits/ceph-helpers.sh:993: wait_for_clean:  cur_active_clean=0
       ../qa/workunits/ceph-helpers.sh:994: wait_for_clean:  test 0 '!=' 0
       ../qa/workunits/ceph-helpers.sh:998: wait_for_clean:  sleep 1
       .../qa/workunits/ceph-helpers.sh:999: wait_for_clean:  expr 21 + 1
       ../qa/workunits/ceph-helpers.sh:999: wait_for_clean:  timer=22
       ../qa/workunits/ceph-helpers.sh:985: wait_for_clean:  is_clean
       .../qa/workunits/ceph-helpers.sh:952: is_clean:  get_num_pgs
       .../qa/workunits/ceph-helpers.sh:898: get_num_pgs:  ceph --format xml status
       .../qa/workunits/ceph-helpers.sh:899: get_num_pgs:  xmlstarlet sel -t -m //pgmap/num_pgs -v .
       ../qa/workunits/ceph-helpers.sh:952: is_clean:  num_pgs=4
       ../qa/workunits/ceph-helpers.sh:953: is_clean:  test 4 '!=' 0
       .../qa/workunits/ceph-helpers.sh:954: is_clean:  get_num_active_clean
       .../qa/workunits/ceph-helpers.sh:863: get_num_active_clean:  local 'expression=('
       .../qa/workunits/ceph-helpers.sh:864: get_num_active_clean:  expression+='contains(.,'\''active'\'') and '
       .../qa/workunits/ceph-helpers.sh:865: get_num_active_clean:  expression+='contains(.,'\''clean'\'') and '
       .../qa/workunits/ceph-helpers.sh:866: get_num_active_clean:  expression+='not(contains(.,'\''stale'\''))'
       .../qa/workunits/ceph-helpers.sh:867: get_num_active_clean:  expression+=')'
       .../qa/workunits/ceph-helpers.sh:871: get_num_active_clean:  ceph --format xml pg dump pgs
       .../qa/workunits/ceph-helpers.sh:873: get_num_active_clean:  grep -v '^$'
       .../qa/workunits/ceph-helpers.sh:872: get_num_active_clean:  xmlstarlet sel -t -m '//pg_stat/state[(contains(.,'\''active'\'') and contains(.,'\''clean'\'') and not(contains(.,'\''stale'\'')))]' -v . -n
       .../qa/workunits/ceph-helpers.sh:873: get_num_active_clean:  wc -l
       ../qa/workunits/ceph-helpers.sh:954: is_clean:  test 0 = 4
       ../qa/workunits/ceph-helpers.sh:954: is_clean:  return 1
       ../qa/workunits/ceph-helpers.sh:986: wait_for_clean:  get_is_making_recovery_progress
       .../qa/workunits/ceph-helpers.sh:839: get_is_making_recovery_progress:  ceph --format xml status
       .../qa/workunits/ceph-helpers.sh:839: get_is_making_recovery_progress:  xmlstarlet sel -t -m //pgmap/recovering_keys_per_sec -v . -o ' ' -t -m //pgmap/recovering_bytes_per_sec -v . -o ' ' -t -m //pgmap/recovering_objects_per_sec -v .
       ../qa/workunits/ceph-helpers.sh:839: get_is_making_recovery_progress:  local progress=
       ../qa/workunits/ceph-helpers.sh:840: get_is_making_recovery_progress:  test -n ''
       ../qa/workunits/ceph-helpers.sh:988: wait_for_clean:  ((  timer >= 120  ))
       .../qa/workunits/ceph-helpers.sh:993: wait_for_clean:  get_num_active_clean
       .../qa/workunits/ceph-helpers.sh:863: get_num_active_clean:  local 'expression=('
       .../qa/workunits/ceph-helpers.sh:864: get_num_active_clean:  expression+='contains(.,'\''active'\'') and '
       .../qa/workunits/ceph-helpers.sh:865: get_num_active_clean:  expression+='contains(.,'\''clean'\'') and '
       .../qa/workunits/ceph-helpers.sh:866: get_num_active_clean:  expression+='not(contains(.,'\''stale'\''))'
       .../qa/workunits/ceph-helpers.sh:867: get_num_active_clean:  expression+=')'
       .../qa/workunits/ceph-helpers.sh:871: get_num_active_clean:  ceph --format xml pg dump pgs
       .../qa/workunits/ceph-helpers.sh:872: get_num_active_clean:  xmlstarlet sel -t -m '//pg_stat/state[(contains(.,'\''active'\'') and contains(.,'\''clean'\'') and not(contains(.,'\''stale'\'')))]' -v . -n
       .../qa/workunits/ceph-helpers.sh:873: get_num_active_clean:  grep -v '^$'
       .../qa/workunits/ceph-helpers.sh:873: get_num_active_clean:  wc -l
       ../qa/workunits/ceph-helpers.sh:993: wait_for_clean:  cur_active_clean=0
       ../qa/workunits/ceph-helpers.sh:994: wait_for_clean:  test 0 '!=' 0
       ../qa/workunits/ceph-helpers.sh:998: wait_for_clean:  sleep 1
       .../qa/workunits/ceph-helpers.sh:999: wait_for_clean:  expr 22 + 1
       ../qa/workunits/ceph-helpers.sh:999: wait_for_clean:  timer=23
       ../qa/workunits/ceph-helpers.sh:985: wait_for_clean:  is_clean
       .../qa/workunits/ceph-helpers.sh:952: is_clean:  get_num_pgs
       .../qa/workunits/ceph-helpers.sh:899: get_num_pgs:  xmlstarlet sel -t -m //pgmap/num_pgs -v .
       .../qa/workunits/ceph-helpers.sh:898: get_num_pgs:  ceph --format xml status
       ../qa/workunits/ceph-helpers.sh:952: is_clean:  num_pgs=4
       ../qa/workunits/ceph-helpers.sh:953: is_clean:  test 4 '!=' 0
       .../qa/workunits/ceph-helpers.sh:954: is_clean:  get_num_active_clean
       .../qa/workunits/ceph-helpers.sh:863: get_num_active_clean:  local 'expression=('
       .../qa/workunits/ceph-helpers.sh:864: get_num_active_clean:  expression+='contains(.,'\''active'\'') and '
       .../qa/workunits/ceph-helpers.sh:865: get_num_active_clean:  expression+='contains(.,'\''clean'\'') and '
       .../qa/workunits/ceph-helpers.sh:866: get_num_active_clean:  expression+='not(contains(.,'\''stale'\''))'
       .../qa/workunits/ceph-helpers.sh:867: get_num_active_clean:  expression+=')'
       .../qa/workunits/ceph-helpers.sh:871: get_num_active_clean:  ceph --format xml pg dump pgs
       .../qa/workunits/ceph-helpers.sh:872: get_num_active_clean:  xmlstarlet sel -t -m '//pg_stat/state[(contains(.,'\''active'\'') and contains(.,'\''clean'\'') and not(contains(.,'\''stale'\'')))]' -v . -n
       .../qa/workunits/ceph-helpers.sh:873: get_num_active_clean:  grep -v '^$'
       .../qa/workunits/ceph-helpers.sh:873: get_num_active_clean:  wc -l
       ../qa/workunits/ceph-helpers.sh:954: is_clean:  test 0 = 4
       ../qa/workunits/ceph-helpers.sh:954: is_clean:  return 1
       ../qa/workunits/ceph-helpers.sh:986: wait_for_clean:  get_is_making_recovery_progress
       .../qa/workunits/ceph-helpers.sh:839: get_is_making_recovery_progress:  ceph --format xml status
       .../qa/workunits/ceph-helpers.sh:839: get_is_making_recovery_progress:  xmlstarlet sel -t -m //pgmap/recovering_keys_per_sec -v . -o ' ' -t -m //pgmap/recovering_bytes_per_sec -v . -o ' ' -t -m //pgmap/recovering_objects_per_sec -v .
       ../qa/workunits/ceph-helpers.sh:839: get_is_making_recovery_progress:  local progress=
       ../qa/workunits/ceph-helpers.sh:840: get_is_making_recovery_progress:  test -n ''
       ../qa/workunits/ceph-helpers.sh:988: wait_for_clean:  ((  timer >= 120  ))
       .../qa/workunits/ceph-helpers.sh:993: wait_for_clean:  get_num_active_clean
       .../qa/workunits/ceph-helpers.sh:863: get_num_active_clean:  local 'expression=('
       .../qa/workunits/ceph-helpers.sh:864: get_num_active_clean:  expression+='contains(.,'\''active'\'') and '
       .../qa/workunits/ceph-helpers.sh:865: get_num_active_clean:  expression+='contains(.,'\''clean'\'') and '
       .../qa/workunits/ceph-helpers.sh:866: get_num_active_clean:  expression+='not(contains(.,'\''stale'\''))'
       .../qa/workunits/ceph-helpers.sh:867: get_num_active_clean:  expression+=')'
       .../qa/workunits/ceph-helpers.sh:871: get_num_active_clean:  ceph --format xml pg dump pgs
       .../qa/workunits/ceph-helpers.sh:872: get_num_active_clean:  xmlstarlet sel -t -m '//pg_stat/state[(contains(.,'\''active'\'') and contains(.,'\''clean'\'') and not(contains(.,'\''stale'\'')))]' -v . -n
       .../qa/workunits/ceph-helpers.sh:873: get_num_active_clean:  grep -v '^$'
       .../qa/workunits/ceph-helpers.sh:873: get_num_active_clean:  wc -l
       ../qa/workunits/ceph-helpers.sh:993: wait_for_clean:  cur_active_clean=0
       ../qa/workunits/ceph-helpers.sh:994: wait_for_clean:  test 0 '!=' 0
       ../qa/workunits/ceph-helpers.sh:998: wait_for_clean:  sleep 1
       .../qa/workunits/ceph-helpers.sh:999: wait_for_clean:  expr 23 + 1
       ../qa/workunits/ceph-helpers.sh:999: wait_for_clean:  timer=24
       ../qa/workunits/ceph-helpers.sh:985: wait_for_clean:  is_clean
       .../qa/workunits/ceph-helpers.sh:952: is_clean:  get_num_pgs
       .../qa/workunits/ceph-helpers.sh:898: get_num_pgs:  ceph --format xml status
       .../qa/workunits/ceph-helpers.sh:899: get_num_pgs:  xmlstarlet sel -t -m //pgmap/num_pgs -v .
       ../qa/workunits/ceph-helpers.sh:952: is_clean:  num_pgs=4
       ../qa/workunits/ceph-helpers.sh:953: is_clean:  test 4 '!=' 0
       .../qa/workunits/ceph-helpers.sh:954: is_clean:  get_num_active_clean
       .../qa/workunits/ceph-helpers.sh:863: get_num_active_clean:  local 'expression=('
       .../qa/workunits/ceph-helpers.sh:864: get_num_active_clean:  expression+='contains(.,'\''active'\'') and '
       .../qa/workunits/ceph-helpers.sh:865: get_num_active_clean:  expression+='contains(.,'\''clean'\'') and '
       .../qa/workunits/ceph-helpers.sh:866: get_num_active_clean:  expression+='not(contains(.,'\''stale'\''))'
       .../qa/workunits/ceph-helpers.sh:867: get_num_active_clean:  expression+=')'
       .../qa/workunits/ceph-helpers.sh:871: get_num_active_clean:  ceph --format xml pg dump pgs
       .../qa/workunits/ceph-helpers.sh:872: get_num_active_clean:  xmlstarlet sel -t -m '//pg_stat/state[(contains(.,'\''active'\'') and contains(.,'\''clean'\'') and not(contains(.,'\''stale'\'')))]' -v . -n
       .../qa/workunits/ceph-helpers.sh:873: get_num_active_clean:  grep -v '^$'
       .../qa/workunits/ceph-helpers.sh:873: get_num_active_clean:  wc -l
       ../qa/workunits/ceph-helpers.sh:954: is_clean:  test 0 = 4
       ../qa/workunits/ceph-helpers.sh:954: is_clean:  return 1
       ../qa/workunits/ceph-helpers.sh:986: wait_for_clean:  get_is_making_recovery_progress
       .../qa/workunits/ceph-helpers.sh:839: get_is_making_recovery_progress:  ceph --format xml status
       .../qa/workunits/ceph-helpers.sh:839: get_is_making_recovery_progress:  xmlstarlet sel -t -m //pgmap/recovering_keys_per_sec -v . -o ' ' -t -m //pgmap/recovering_bytes_per_sec -v . -o ' ' -t -m //pgmap/recovering_objects_per_sec -v .
       ../qa/workunits/ceph-helpers.sh:839: get_is_making_recovery_progress:  local progress=
       ../qa/workunits/ceph-helpers.sh:840: get_is_making_recovery_progress:  test -n ''
       ../qa/workunits/ceph-helpers.sh:988: wait_for_clean:  ((  timer >= 120  ))
       .../qa/workunits/ceph-helpers.sh:993: wait_for_clean:  get_num_active_clean
       .../qa/workunits/ceph-helpers.sh:863: get_num_active_clean:  local 'expression=('
       .../qa/workunits/ceph-helpers.sh:864: get_num_active_clean:  expression+='contains(.,'\''active'\'') and '
       .../qa/workunits/ceph-helpers.sh:865: get_num_active_clean:  expression+='contains(.,'\''clean'\'') and '
       .../qa/workunits/ceph-helpers.sh:866: get_num_active_clean:  expression+='not(contains(.,'\''stale'\''))'
       .../qa/workunits/ceph-helpers.sh:867: get_num_active_clean:  expression+=')'
       .../qa/workunits/ceph-helpers.sh:871: get_num_active_clean:  ceph --format xml pg dump pgs
       .../qa/workunits/ceph-helpers.sh:872: get_num_active_clean:  xmlstarlet sel -t -m '//pg_stat/state[(contains(.,'\''active'\'') and contains(.,'\''clean'\'') and not(contains(.,'\''stale'\'')))]' -v . -n
       .../qa/workunits/ceph-helpers.sh:873: get_num_active_clean:  wc -l
       .../qa/workunits/ceph-helpers.sh:873: get_num_active_clean:  grep -v '^$'
       ../qa/workunits/ceph-helpers.sh:993: wait_for_clean:  cur_active_clean=0
       ../qa/workunits/ceph-helpers.sh:994: wait_for_clean:  test 0 '!=' 0
       ../qa/workunits/ceph-helpers.sh:998: wait_for_clean:  sleep 1
       .../qa/workunits/ceph-helpers.sh:999: wait_for_clean:  expr 24 + 1
       ../qa/workunits/ceph-helpers.sh:999: wait_for_clean:  timer=25
       ../qa/workunits/ceph-helpers.sh:985: wait_for_clean:  is_clean
       .../qa/workunits/ceph-helpers.sh:952: is_clean:  get_num_pgs
       .../qa/workunits/ceph-helpers.sh:898: get_num_pgs:  ceph --format xml status
       .../qa/workunits/ceph-helpers.sh:899: get_num_pgs:  xmlstarlet sel -t -m //pgmap/num_pgs -v .
       ../qa/workunits/ceph-helpers.sh:952: is_clean:  num_pgs=4
       ../qa/workunits/ceph-helpers.sh:953: is_clean:  test 4 '!=' 0
       .../qa/workunits/ceph-helpers.sh:954: is_clean:  get_num_active_clean
       .../qa/workunits/ceph-helpers.sh:863: get_num_active_clean:  local 'expression=('
       .../qa/workunits/ceph-helpers.sh:864: get_num_active_clean:  expression+='contains(.,'\''active'\'') and '
       .../qa/workunits/ceph-helpers.sh:865: get_num_active_clean:  expression+='contains(.,'\''clean'\'') and '
       .../qa/workunits/ceph-helpers.sh:866: get_num_active_clean:  expression+='not(contains(.,'\''stale'\''))'
       .../qa/workunits/ceph-helpers.sh:867: get_num_active_clean:  expression+=')'
       .../qa/workunits/ceph-helpers.sh:871: get_num_active_clean:  ceph --format xml pg dump pgs
       .../qa/workunits/ceph-helpers.sh:872: get_num_active_clean:  xmlstarlet sel -t -m '//pg_stat/state[(contains(.,'\''active'\'') and contains(.,'\''clean'\'') and not(contains(.,'\''stale'\'')))]' -v . -n
       .../qa/workunits/ceph-helpers.sh:873: get_num_active_clean:  grep -v '^$'
       .../qa/workunits/ceph-helpers.sh:873: get_num_active_clean:  wc -l
       ../qa/workunits/ceph-helpers.sh:954: is_clean:  test 0 = 4
       ../qa/workunits/ceph-helpers.sh:954: is_clean:  return 1
       ../qa/workunits/ceph-helpers.sh:986: wait_for_clean:  get_is_making_recovery_progress
       .../qa/workunits/ceph-helpers.sh:839: get_is_making_recovery_progress:  ceph --format xml status
       .../qa/workunits/ceph-helpers.sh:839: get_is_making_recovery_progress:  xmlstarlet sel -t -m //pgmap/recovering_keys_per_sec -v . -o ' ' -t -m //pgmap/recovering_bytes_per_sec -v . -o ' ' -t -m //pgmap/recovering_objects_per_sec -v .
       ../qa/workunits/ceph-helpers.sh:839: get_is_making_recovery_progress:  local progress=
       ../qa/workunits/ceph-helpers.sh:840: get_is_making_recovery_progress:  test -n ''
       ../qa/workunits/ceph-helpers.sh:988: wait_for_clean:  ((  timer >= 120  ))
       .../qa/workunits/ceph-helpers.sh:993: wait_for_clean:  get_num_active_clean
       .../qa/workunits/ceph-helpers.sh:863: get_num_active_clean:  local 'expression=('
       .../qa/workunits/ceph-helpers.sh:864: get_num_active_clean:  expression+='contains(.,'\''active'\'') and '
       .../qa/workunits/ceph-helpers.sh:865: get_num_active_clean:  expression+='contains(.,'\''clean'\'') and '
       .../qa/workunits/ceph-helpers.sh:866: get_num_active_clean:  expression+='not(contains(.,'\''stale'\''))'
       .../qa/workunits/ceph-helpers.sh:867: get_num_active_clean:  expression+=')'
       .../qa/workunits/ceph-helpers.sh:871: get_num_active_clean:  ceph --format xml pg dump pgs
       .../qa/workunits/ceph-helpers.sh:873: get_num_active_clean:  wc -l
       .../qa/workunits/ceph-helpers.sh:872: get_num_active_clean:  xmlstarlet sel -t -m '//pg_stat/state[(contains(.,'\''active'\'') and contains(.,'\''clean'\'') and not(contains(.,'\''stale'\'')))]' -v . -n
       .../qa/workunits/ceph-helpers.sh:873: get_num_active_clean:  grep -v '^$'
       ../qa/workunits/ceph-helpers.sh:993: wait_for_clean:  cur_active_clean=0
       ../qa/workunits/ceph-helpers.sh:994: wait_for_clean:  test 0 '!=' 0
       ../qa/workunits/ceph-helpers.sh:998: wait_for_clean:  sleep 1
       .../qa/workunits/ceph-helpers.sh:999: wait_for_clean:  expr 25 + 1
       ../qa/workunits/ceph-helpers.sh:999: wait_for_clean:  timer=26
       ../qa/workunits/ceph-helpers.sh:985: wait_for_clean:  is_clean
       .../qa/workunits/ceph-helpers.sh:952: is_clean:  get_num_pgs
       .../qa/workunits/ceph-helpers.sh:898: get_num_pgs:  ceph --format xml status
       .../qa/workunits/ceph-helpers.sh:899: get_num_pgs:  xmlstarlet sel -t -m //pgmap/num_pgs -v .
       ../qa/workunits/ceph-helpers.sh:952: is_clean:  num_pgs=4
       ../qa/workunits/ceph-helpers.sh:953: is_clean:  test 4 '!=' 0
       .../qa/workunits/ceph-helpers.sh:954: is_clean:  get_num_active_clean
       .../qa/workunits/ceph-helpers.sh:863: get_num_active_clean:  local 'expression=('
       .../qa/workunits/ceph-helpers.sh:864: get_num_active_clean:  expression+='contains(.,'\''active'\'') and '
       .../qa/workunits/ceph-helpers.sh:865: get_num_active_clean:  expression+='contains(.,'\''clean'\'') and '
       .../qa/workunits/ceph-helpers.sh:866: get_num_active_clean:  expression+='not(contains(.,'\''stale'\''))'
       .../qa/workunits/ceph-helpers.sh:867: get_num_active_clean:  expression+=')'
       .../qa/workunits/ceph-helpers.sh:872: get_num_active_clean:  xmlstarlet sel -t -m '//pg_stat/state[(contains(.,'\''active'\'') and contains(.,'\''clean'\'') and not(contains(.,'\''stale'\'')))]' -v . -n
       .../qa/workunits/ceph-helpers.sh:873: get_num_active_clean:  grep -v '^$'
       .../qa/workunits/ceph-helpers.sh:873: get_num_active_clean:  wc -l
       .../qa/workunits/ceph-helpers.sh:871: get_num_active_clean:  ceph --format xml pg dump pgs
       ../qa/workunits/ceph-helpers.sh:954: is_clean:  test 0 = 4
       ../qa/workunits/ceph-helpers.sh:954: is_clean:  return 1
       ../qa/workunits/ceph-helpers.sh:986: wait_for_clean:  get_is_making_recovery_progress
       .../qa/workunits/ceph-helpers.sh:839: get_is_making_recovery_progress:  ceph --format xml status
       .../qa/workunits/ceph-helpers.sh:839: get_is_making_recovery_progress:  xmlstarlet sel -t -m //pgmap/recovering_keys_per_sec -v . -o ' ' -t -m //pgmap/recovering_bytes_per_sec -v . -o ' ' -t -m //pgmap/recovering_objects_per_sec -v .
       ../qa/workunits/ceph-helpers.sh:839: get_is_making_recovery_progress:  local progress=
       ../qa/workunits/ceph-helpers.sh:840: get_is_making_recovery_progress:  test -n ''
       ../qa/workunits/ceph-helpers.sh:988: wait_for_clean:  ((  timer >= 120  ))
       .../qa/workunits/ceph-helpers.sh:993: wait_for_clean:  get_num_active_clean
       .../qa/workunits/ceph-helpers.sh:863: get_num_active_clean:  local 'expression=('
       .../qa/workunits/ceph-helpers.sh:864: get_num_active_clean:  expression+='contains(.,'\''active'\'') and '
       .../qa/workunits/ceph-helpers.sh:865: get_num_active_clean:  expression+='contains(.,'\''clean'\'') and '
       .../qa/workunits/ceph-helpers.sh:866: get_num_active_clean:  expression+='not(contains(.,'\''stale'\''))'
       .../qa/workunits/ceph-helpers.sh:867: get_num_active_clean:  expression+=')'
       .../qa/workunits/ceph-helpers.sh:871: get_num_active_clean:  ceph --format xml pg dump pgs
       .../qa/workunits/ceph-helpers.sh:872: get_num_active_clean:  xmlstarlet sel -t -m '//pg_stat/state[(contains(.,'\''active'\'') and contains(.,'\''clean'\'') and not(contains(.,'\''stale'\'')))]' -v . -n
       .../qa/workunits/ceph-helpers.sh:873: get_num_active_clean:  grep -v '^$'
       .../qa/workunits/ceph-helpers.sh:873: get_num_active_clean:  wc -l
       ../qa/workunits/ceph-helpers.sh:993: wait_for_clean:  cur_active_clean=0
       ../qa/workunits/ceph-helpers.sh:994: wait_for_clean:  test 0 '!=' 0
       ../qa/workunits/ceph-helpers.sh:998: wait_for_clean:  sleep 1
       .../qa/workunits/ceph-helpers.sh:999: wait_for_clean:  expr 26 + 1
       ../qa/workunits/ceph-helpers.sh:999: wait_for_clean:  timer=27
       ../qa/workunits/ceph-helpers.sh:985: wait_for_clean:  is_clean
       .../qa/workunits/ceph-helpers.sh:952: is_clean:  get_num_pgs
       .../qa/workunits/ceph-helpers.sh:898: get_num_pgs:  ceph --format xml status
       .../qa/workunits/ceph-helpers.sh:899: get_num_pgs:  xmlstarlet sel -t -m //pgmap/num_pgs -v .
       ../qa/workunits/ceph-helpers.sh:952: is_clean:  num_pgs=4
       ../qa/workunits/ceph-helpers.sh:953: is_clean:  test 4 '!=' 0
       .../qa/workunits/ceph-helpers.sh:954: is_clean:  get_num_active_clean
       .../qa/workunits/ceph-helpers.sh:863: get_num_active_clean:  local 'expression=('
       .../qa/workunits/ceph-helpers.sh:864: get_num_active_clean:  expression+='contains(.,'\''active'\'') and '
       .../qa/workunits/ceph-helpers.sh:865: get_num_active_clean:  expression+='contains(.,'\''clean'\'') and '
       .../qa/workunits/ceph-helpers.sh:866: get_num_active_clean:  expression+='not(contains(.,'\''stale'\''))'
       .../qa/workunits/ceph-helpers.sh:867: get_num_active_clean:  expression+=')'
       .../qa/workunits/ceph-helpers.sh:871: get_num_active_clean:  ceph --format xml pg dump pgs
       .../qa/workunits/ceph-helpers.sh:872: get_num_active_clean:  xmlstarlet sel -t -m '//pg_stat/state[(contains(.,'\''active'\'') and contains(.,'\''clean'\'') and not(contains(.,'\''stale'\'')))]' -v . -n
       .../qa/workunits/ceph-helpers.sh:873: get_num_active_clean:  grep -v '^$'
       .../qa/workunits/ceph-helpers.sh:873: get_num_active_clean:  wc -l
       ../qa/workunits/ceph-helpers.sh:954: is_clean:  test 0 = 4
       ../qa/workunits/ceph-helpers.sh:954: is_clean:  return 1
       ../qa/workunits/ceph-helpers.sh:986: wait_for_clean:  get_is_making_recovery_progress
       .../qa/workunits/ceph-helpers.sh:839: get_is_making_recovery_progress:  ceph --format xml status
       .../qa/workunits/ceph-helpers.sh:839: get_is_making_recovery_progress:  xmlstarlet sel -t -m //pgmap/recovering_keys_per_sec -v . -o ' ' -t -m //pgmap/recovering_bytes_per_sec -v . -o ' ' -t -m //pgmap/recovering_objects_per_sec -v .
       ../qa/workunits/ceph-helpers.sh:839: get_is_making_recovery_progress:  local progress=
       ../qa/workunits/ceph-helpers.sh:840: get_is_making_recovery_progress:  test -n ''
       ../qa/workunits/ceph-helpers.sh:988: wait_for_clean:  ((  timer >= 120  ))
       .../qa/workunits/ceph-helpers.sh:993: wait_for_clean:  get_num_active_clean
       .../qa/workunits/ceph-helpers.sh:863: get_num_active_clean:  local 'expression=('
       .../qa/workunits/ceph-helpers.sh:864: get_num_active_clean:  expression+='contains(.,'\''active'\'') and '
       .../qa/workunits/ceph-helpers.sh:865: get_num_active_clean:  expression+='contains(.,'\''clean'\'') and '
       .../qa/workunits/ceph-helpers.sh:866: get_num_active_clean:  expression+='not(contains(.,'\''stale'\''))'
       .../qa/workunits/ceph-helpers.sh:867: get_num_active_clean:  expression+=')'
       .../qa/workunits/ceph-helpers.sh:871: get_num_active_clean:  ceph --format xml pg dump pgs
       .../qa/workunits/ceph-helpers.sh:873: get_num_active_clean:  grep -v '^$'
       .../qa/workunits/ceph-helpers.sh:873: get_num_active_clean:  wc -l
       .../qa/workunits/ceph-helpers.sh:872: get_num_active_clean:  xmlstarlet sel -t -m '//pg_stat/state[(contains(.,'\''active'\'') and contains(.,'\''clean'\'') and not(contains(.,'\''stale'\'')))]' -v . -n
       ../qa/workunits/ceph-helpers.sh:993: wait_for_clean:  cur_active_clean=0
       ../qa/workunits/ceph-helpers.sh:994: wait_for_clean:  test 0 '!=' 0
       ../qa/workunits/ceph-helpers.sh:998: wait_for_clean:  sleep 1
       .../qa/workunits/ceph-helpers.sh:999: wait_for_clean:  expr 27 + 1
       ../qa/workunits/ceph-helpers.sh:999: wait_for_clean:  timer=28
       ../qa/workunits/ceph-helpers.sh:985: wait_for_clean:  is_clean
       .../qa/workunits/ceph-helpers.sh:952: is_clean:  get_num_pgs
       .../qa/workunits/ceph-helpers.sh:898: get_num_pgs:  ceph --format xml status
       .../qa/workunits/ceph-helpers.sh:899: get_num_pgs:  xmlstarlet sel -t -m //pgmap/num_pgs -v .
       ../qa/workunits/ceph-helpers.sh:952: is_clean:  num_pgs=4
       ../qa/workunits/ceph-helpers.sh:953: is_clean:  test 4 '!=' 0
       .../qa/workunits/ceph-helpers.sh:954: is_clean:  get_num_active_clean
       .../qa/workunits/ceph-helpers.sh:863: get_num_active_clean:  local 'expression=('
       .../qa/workunits/ceph-helpers.sh:864: get_num_active_clean:  expression+='contains(.,'\''active'\'') and '
       .../qa/workunits/ceph-helpers.sh:865: get_num_active_clean:  expression+='contains(.,'\''clean'\'') and '
       .../qa/workunits/ceph-helpers.sh:866: get_num_active_clean:  expression+='not(contains(.,'\''stale'\''))'
       .../qa/workunits/ceph-helpers.sh:867: get_num_active_clean:  expression+=')'
       .../qa/workunits/ceph-helpers.sh:871: get_num_active_clean:  ceph --format xml pg dump pgs
       .../qa/workunits/ceph-helpers.sh:872: get_num_active_clean:  xmlstarlet sel -t -m '//pg_stat/state[(contains(.,'\''active'\'') and contains(.,'\''clean'\'') and not(contains(.,'\''stale'\'')))]' -v . -n
       .../qa/workunits/ceph-helpers.sh:873: get_num_active_clean:  wc -l
       .../qa/workunits/ceph-helpers.sh:873: get_num_active_clean:  grep -v '^$'
       ../qa/workunits/ceph-helpers.sh:954: is_clean:  test 0 = 4
       ../qa/workunits/ceph-helpers.sh:954: is_clean:  return 1
       ../qa/workunits/ceph-helpers.sh:986: wait_for_clean:  get_is_making_recovery_progress
       .../qa/workunits/ceph-helpers.sh:839: get_is_making_recovery_progress:  ceph --format xml status
       .../qa/workunits/ceph-helpers.sh:839: get_is_making_recovery_progress:  xmlstarlet sel -t -m //pgmap/recovering_keys_per_sec -v . -o ' ' -t -m //pgmap/recovering_bytes_per_sec -v . -o ' ' -t -m //pgmap/recovering_objects_per_sec -v .
       ../qa/workunits/ceph-helpers.sh:839: get_is_making_recovery_progress:  local progress=
       ../qa/workunits/ceph-helpers.sh:840: get_is_making_recovery_progress:  test -n ''
       ../qa/workunits/ceph-helpers.sh:988: wait_for_clean:  ((  timer >= 120  ))
       .../qa/workunits/ceph-helpers.sh:993: wait_for_clean:  get_num_active_clean
       .../qa/workunits/ceph-helpers.sh:863: get_num_active_clean:  local 'expression=('
       .../qa/workunits/ceph-helpers.sh:864: get_num_active_clean:  expression+='contains(.,'\''active'\'') and '
       .../qa/workunits/ceph-helpers.sh:865: get_num_active_clean:  expression+='contains(.,'\''clean'\'') and '
       .../qa/workunits/ceph-helpers.sh:866: get_num_active_clean:  expression+='not(contains(.,'\''stale'\''))'
       .../qa/workunits/ceph-helpers.sh:867: get_num_active_clean:  expression+=')'
       .../qa/workunits/ceph-helpers.sh:871: get_num_active_clean:  ceph --format xml pg dump pgs
       .../qa/workunits/ceph-helpers.sh:873: get_num_active_clean:  grep -v '^$'
       .../qa/workunits/ceph-helpers.sh:872: get_num_active_clean:  xmlstarlet sel -t -m '//pg_stat/state[(contains(.,'\''active'\'') and contains(.,'\''clean'\'') and not(contains(.,'\''stale'\'')))]' -v . -n
       .../qa/workunits/ceph-helpers.sh:873: get_num_active_clean:  wc -l
       ../qa/workunits/ceph-helpers.sh:993: wait_for_clean:  cur_active_clean=0
       ../qa/workunits/ceph-helpers.sh:994: wait_for_clean:  test 0 '!=' 0
       ../qa/workunits/ceph-helpers.sh:998: wait_for_clean:  sleep 1
       .../qa/workunits/ceph-helpers.sh:999: wait_for_clean:  expr 28 + 1
       ../qa/workunits/ceph-helpers.sh:999: wait_for_clean:  timer=29
       ../qa/workunits/ceph-helpers.sh:985: wait_for_clean:  is_clean
       .../qa/workunits/ceph-helpers.sh:952: is_clean:  get_num_pgs
       .../qa/workunits/ceph-helpers.sh:898: get_num_pgs:  ceph --format xml status
       .../qa/workunits/ceph-helpers.sh:899: get_num_pgs:  xmlstarlet sel -t -m //pgmap/num_pgs -v .
       ../qa/workunits/ceph-helpers.sh:952: is_clean:  num_pgs=4
       ../qa/workunits/ceph-helpers.sh:953: is_clean:  test 4 '!=' 0
       .../qa/workunits/ceph-helpers.sh:954: is_clean:  get_num_active_clean
       .../qa/workunits/ceph-helpers.sh:863: get_num_active_clean:  local 'expression=('
       .../qa/workunits/ceph-helpers.sh:864: get_num_active_clean:  expression+='contains(.,'\''active'\'') and '
       .../qa/workunits/ceph-helpers.sh:865: get_num_active_clean:  expression+='contains(.,'\''clean'\'') and '
       .../qa/workunits/ceph-helpers.sh:866: get_num_active_clean:  expression+='not(contains(.,'\''stale'\''))'
       .../qa/workunits/ceph-helpers.sh:867: get_num_active_clean:  expression+=')'
       .../qa/workunits/ceph-helpers.sh:871: get_num_active_clean:  ceph --format xml pg dump pgs
       .../qa/workunits/ceph-helpers.sh:872: get_num_active_clean:  xmlstarlet sel -t -m '//pg_stat/state[(contains(.,'\''active'\'') and contains(.,'\''clean'\'') and not(contains(.,'\''stale'\'')))]' -v . -n
       .../qa/workunits/ceph-helpers.sh:873: get_num_active_clean:  grep -v '^$'
       .../qa/workunits/ceph-helpers.sh:873: get_num_active_clean:  wc -l
       ../qa/workunits/ceph-helpers.sh:954: is_clean:  test 0 = 4
       ../qa/workunits/ceph-helpers.sh:954: is_clean:  return 1
       ../qa/workunits/ceph-helpers.sh:986: wait_for_clean:  get_is_making_recovery_progress
       .../qa/workunits/ceph-helpers.sh:839: get_is_making_recovery_progress:  ceph --format xml status
       .../qa/workunits/ceph-helpers.sh:839: get_is_making_recovery_progress:  xmlstarlet sel -t -m //pgmap/recovering_keys_per_sec -v . -o ' ' -t -m //pgmap/recovering_bytes_per_sec -v . -o ' ' -t -m //pgmap/recovering_objects_per_sec -v .
       ../qa/workunits/ceph-helpers.sh:839: get_is_making_recovery_progress:  local progress=
       ../qa/workunits/ceph-helpers.sh:840: get_is_making_recovery_progress:  test -n ''
       ../qa/workunits/ceph-helpers.sh:988: wait_for_clean:  ((  timer >= 120  ))
       .../qa/workunits/ceph-helpers.sh:993: wait_for_clean:  get_num_active_clean
       .../qa/workunits/ceph-helpers.sh:863: get_num_active_clean:  local 'expression=('
       .../qa/workunits/ceph-helpers.sh:864: get_num_active_clean:  expression+='contains(.,'\''active'\'') and '
       .../qa/workunits/ceph-helpers.sh:865: get_num_active_clean:  expression+='contains(.,'\''clean'\'') and '
       .../qa/workunits/ceph-helpers.sh:866: get_num_active_clean:  expression+='not(contains(.,'\''stale'\''))'
       .../qa/workunits/ceph-helpers.sh:867: get_num_active_clean:  expression+=')'
       .../qa/workunits/ceph-helpers.sh:871: get_num_active_clean:  ceph --format xml pg dump pgs
       .../qa/workunits/ceph-helpers.sh:872: get_num_active_clean:  xmlstarlet sel -t -m '//pg_stat/state[(contains(.,'\''active'\'') and contains(.,'\''clean'\'') and not(contains(.,'\''stale'\'')))]' -v . -n
       .../qa/workunits/ceph-helpers.sh:873: get_num_active_clean:  grep -v '^$'
       .../qa/workunits/ceph-helpers.sh:873: get_num_active_clean:  wc -l
       ../qa/workunits/ceph-helpers.sh:993: wait_for_clean:  cur_active_clean=0
       ../qa/workunits/ceph-helpers.sh:994: wait_for_clean:  test 0 '!=' 0
       ../qa/workunits/ceph-helpers.sh:998: wait_for_clean:  sleep 1
       .../qa/workunits/ceph-helpers.sh:999: wait_for_clean:  expr 29 + 1
       ../qa/workunits/ceph-helpers.sh:999: wait_for_clean:  timer=30
       ../qa/workunits/ceph-helpers.sh:985: wait_for_clean:  is_clean
       .../qa/workunits/ceph-helpers.sh:952: is_clean:  get_num_pgs
       .../qa/workunits/ceph-helpers.sh:898: get_num_pgs:  ceph --format xml status
       .../qa/workunits/ceph-helpers.sh:899: get_num_pgs:  xmlstarlet sel -t -m //pgmap/num_pgs -v .
       ../qa/workunits/ceph-helpers.sh:952: is_clean:  num_pgs=4
       ../qa/workunits/ceph-helpers.sh:953: is_clean:  test 4 '!=' 0
       .../qa/workunits/ceph-helpers.sh:954: is_clean:  get_num_active_clean
       .../qa/workunits/ceph-helpers.sh:863: get_num_active_clean:  local 'expression=('
       .../qa/workunits/ceph-helpers.sh:864: get_num_active_clean:  expression+='contains(.,'\''active'\'') and '
       .../qa/workunits/ceph-helpers.sh:865: get_num_active_clean:  expression+='contains(.,'\''clean'\'') and '
       .../qa/workunits/ceph-helpers.sh:866: get_num_active_clean:  expression+='not(contains(.,'\''stale'\''))'
       .../qa/workunits/ceph-helpers.sh:867: get_num_active_clean:  expression+=')'
       .../qa/workunits/ceph-helpers.sh:871: get_num_active_clean:  ceph --format xml pg dump pgs
       .../qa/workunits/ceph-helpers.sh:872: get_num_active_clean:  xmlstarlet sel -t -m '//pg_stat/state[(contains(.,'\''active'\'') and contains(.,'\''clean'\'') and not(contains(.,'\''stale'\'')))]' -v . -n
       .../qa/workunits/ceph-helpers.sh:873: get_num_active_clean:  grep -v '^$'
       .../qa/workunits/ceph-helpers.sh:873: get_num_active_clean:  wc -l
       ../qa/workunits/ceph-helpers.sh:954: is_clean:  test 0 = 4
       ../qa/workunits/ceph-helpers.sh:954: is_clean:  return 1
       ../qa/workunits/ceph-helpers.sh:986: wait_for_clean:  get_is_making_recovery_progress
       .../qa/workunits/ceph-helpers.sh:839: get_is_making_recovery_progress:  ceph --format xml status
       .../qa/workunits/ceph-helpers.sh:839: get_is_making_recovery_progress:  xmlstarlet sel -t -m //pgmap/recovering_keys_per_sec -v . -o ' ' -t -m //pgmap/recovering_bytes_per_sec -v . -o ' ' -t -m //pgmap/recovering_objects_per_sec -v .
       ../qa/workunits/ceph-helpers.sh:839: get_is_making_recovery_progress:  local progress=
       ../qa/workunits/ceph-helpers.sh:840: get_is_making_recovery_progress:  test -n ''
       ../qa/workunits/ceph-helpers.sh:988: wait_for_clean:  ((  timer >= 120  ))
       .../qa/workunits/ceph-helpers.sh:993: wait_for_clean:  get_num_active_clean
       .../qa/workunits/ceph-helpers.sh:863: get_num_active_clean:  local 'expression=('
       .../qa/workunits/ceph-helpers.sh:864: get_num_active_clean:  expression+='contains(.,'\''active'\'') and '
       .../qa/workunits/ceph-helpers.sh:865: get_num_active_clean:  expression+='contains(.,'\''clean'\'') and '
       .../qa/workunits/ceph-helpers.sh:866: get_num_active_clean:  expression+='not(contains(.,'\''stale'\''))'
       .../qa/workunits/ceph-helpers.sh:867: get_num_active_clean:  expression+=')'
       .../qa/workunits/ceph-helpers.sh:871: get_num_active_clean:  ceph --format xml pg dump pgs
       .../qa/workunits/ceph-helpers.sh:872: get_num_active_clean:  xmlstarlet sel -t -m '//pg_stat/state[(contains(.,'\''active'\'') and contains(.,'\''clean'\'') and not(contains(.,'\''stale'\'')))]' -v . -n
       .../qa/workunits/ceph-helpers.sh:873: get_num_active_clean:  grep -v '^$'
       .../qa/workunits/ceph-helpers.sh:873: get_num_active_clean:  wc -l
       ../qa/workunits/ceph-helpers.sh:993: wait_for_clean:  cur_active_clean=0
       ../qa/workunits/ceph-helpers.sh:994: wait_for_clean:  test 0 '!=' 0
       ../qa/workunits/ceph-helpers.sh:998: wait_for_clean:  sleep 1
       .../qa/workunits/ceph-helpers.sh:999: wait_for_clean:  expr 30 + 1
       ../qa/workunits/ceph-helpers.sh:999: wait_for_clean:  timer=31
       ../qa/workunits/ceph-helpers.sh:985: wait_for_clean:  is_clean
       .../qa/workunits/ceph-helpers.sh:952: is_clean:  get_num_pgs
       .../qa/workunits/ceph-helpers.sh:898: get_num_pgs:  ceph --format xml status
       .../qa/workunits/ceph-helpers.sh:899: get_num_pgs:  xmlstarlet sel -t -m //pgmap/num_pgs -v .
       ../qa/workunits/ceph-helpers.sh:952: is_clean:  num_pgs=4
       ../qa/workunits/ceph-helpers.sh:953: is_clean:  test 4 '!=' 0
       .../qa/workunits/ceph-helpers.sh:954: is_clean:  get_num_active_clean
       .../qa/workunits/ceph-helpers.sh:863: get_num_active_clean:  local 'expression=('
       .../qa/workunits/ceph-helpers.sh:864: get_num_active_clean:  expression+='contains(.,'\''active'\'') and '
       .../qa/workunits/ceph-helpers.sh:865: get_num_active_clean:  expression+='contains(.,'\''clean'\'') and '
       .../qa/workunits/ceph-helpers.sh:866: get_num_active_clean:  expression+='not(contains(.,'\''stale'\''))'
       .../qa/workunits/ceph-helpers.sh:867: get_num_active_clean:  expression+=')'
       .../qa/workunits/ceph-helpers.sh:871: get_num_active_clean:  ceph --format xml pg dump pgs
       .../qa/workunits/ceph-helpers.sh:872: get_num_active_clean:  xmlstarlet sel -t -m '//pg_stat/state[(contains(.,'\''active'\'') and contains(.,'\''clean'\'') and not(contains(.,'\''stale'\'')))]' -v . -n
       .../qa/workunits/ceph-helpers.sh:873: get_num_active_clean:  grep -v '^$'
       .../qa/workunits/ceph-helpers.sh:873: get_num_active_clean:  wc -l
       ../qa/workunits/ceph-helpers.sh:954: is_clean:  test 0 = 4
       ../qa/workunits/ceph-helpers.sh:954: is_clean:  return 1
       ../qa/workunits/ceph-helpers.sh:986: wait_for_clean:  get_is_making_recovery_progress
       .../qa/workunits/ceph-helpers.sh:839: get_is_making_recovery_progress:  ceph --format xml status
       .../qa/workunits/ceph-helpers.sh:839: get_is_making_recovery_progress:  xmlstarlet sel -t -m //pgmap/recovering_keys_per_sec -v . -o ' ' -t -m //pgmap/recovering_bytes_per_sec -v . -o ' ' -t -m //pgmap/recovering_objects_per_sec -v .
       ../qa/workunits/ceph-helpers.sh:839: get_is_making_recovery_progress:  local progress=
       ../qa/workunits/ceph-helpers.sh:840: get_is_making_recovery_progress:  test -n ''
       ../qa/workunits/ceph-helpers.sh:988: wait_for_clean:  ((  timer >= 120  ))
       .../qa/workunits/ceph-helpers.sh:993: wait_for_clean:  get_num_active_clean
       .../qa/workunits/ceph-helpers.sh:863: get_num_active_clean:  local 'expression=('
       .../qa/workunits/ceph-helpers.sh:864: get_num_active_clean:  expression+='contains(.,'\''active'\'') and '
       .../qa/workunits/ceph-helpers.sh:865: get_num_active_clean:  expression+='contains(.,'\''clean'\'') and '
       .../qa/workunits/ceph-helpers.sh:866: get_num_active_clean:  expression+='not(contains(.,'\''stale'\''))'
       .../qa/workunits/ceph-helpers.sh:867: get_num_active_clean:  expression+=')'
       .../qa/workunits/ceph-helpers.sh:871: get_num_active_clean:  ceph --format xml pg dump pgs
       .../qa/workunits/ceph-helpers.sh:873: get_num_active_clean:  grep -v '^$'
       .../qa/workunits/ceph-helpers.sh:872: get_num_active_clean:  xmlstarlet sel -t -m '//pg_stat/state[(contains(.,'\''active'\'') and contains(.,'\''clean'\'') and not(contains(.,'\''stale'\'')))]' -v . -n
       .../qa/workunits/ceph-helpers.sh:873: get_num_active_clean:  wc -l
       ../qa/workunits/ceph-helpers.sh:993: wait_for_clean:  cur_active_clean=0
       ../qa/workunits/ceph-helpers.sh:994: wait_for_clean:  test 0 '!=' 0
       ../qa/workunits/ceph-helpers.sh:998: wait_for_clean:  sleep 1
       .../qa/workunits/ceph-helpers.sh:999: wait_for_clean:  expr 31 + 1
       ../qa/workunits/ceph-helpers.sh:999: wait_for_clean:  timer=32
       ../qa/workunits/ceph-helpers.sh:985: wait_for_clean:  is_clean
       .../qa/workunits/ceph-helpers.sh:952: is_clean:  get_num_pgs
       .../qa/workunits/ceph-helpers.sh:898: get_num_pgs:  ceph --format xml status
       .../qa/workunits/ceph-helpers.sh:899: get_num_pgs:  xmlstarlet sel -t -m //pgmap/num_pgs -v .
       ../qa/workunits/ceph-helpers.sh:952: is_clean:  num_pgs=4
       ../qa/workunits/ceph-helpers.sh:953: is_clean:  test 4 '!=' 0
       .../qa/workunits/ceph-helpers.sh:954: is_clean:  get_num_active_clean
       .../qa/workunits/ceph-helpers.sh:863: get_num_active_clean:  local 'expression=('
       .../qa/workunits/ceph-helpers.sh:864: get_num_active_clean:  expression+='contains(.,'\''active'\'') and '
       .../qa/workunits/ceph-helpers.sh:865: get_num_active_clean:  expression+='contains(.,'\''clean'\'') and '
       .../qa/workunits/ceph-helpers.sh:866: get_num_active_clean:  expression+='not(contains(.,'\''stale'\''))'
       .../qa/workunits/ceph-helpers.sh:867: get_num_active_clean:  expression+=')'
       .../qa/workunits/ceph-helpers.sh:871: get_num_active_clean:  ceph --format xml pg dump pgs
       .../qa/workunits/ceph-helpers.sh:872: get_num_active_clean:  xmlstarlet sel -t -m '//pg_stat/state[(contains(.,'\''active'\'') and contains(.,'\''clean'\'') and not(contains(.,'\''stale'\'')))]' -v . -n
       .../qa/workunits/ceph-helpers.sh:873: get_num_active_clean:  wc -l
       .../qa/workunits/ceph-helpers.sh:873: get_num_active_clean:  grep -v '^$'
       ../qa/workunits/ceph-helpers.sh:954: is_clean:  test 0 = 4
       ../qa/workunits/ceph-helpers.sh:954: is_clean:  return 1
       ../qa/workunits/ceph-helpers.sh:986: wait_for_clean:  get_is_making_recovery_progress
       .../qa/workunits/ceph-helpers.sh:839: get_is_making_recovery_progress:  ceph --format xml status
       .../qa/workunits/ceph-helpers.sh:839: get_is_making_recovery_progress:  xmlstarlet sel -t -m //pgmap/recovering_keys_per_sec -v . -o ' ' -t -m //pgmap/recovering_bytes_per_sec -v . -o ' ' -t -m //pgmap/recovering_objects_per_sec -v .
       ../qa/workunits/ceph-helpers.sh:839: get_is_making_recovery_progress:  local progress=
       ../qa/workunits/ceph-helpers.sh:840: get_is_making_recovery_progress:  test -n ''
       ../qa/workunits/ceph-helpers.sh:988: wait_for_clean:  ((  timer >= 120  ))
       .../qa/workunits/ceph-helpers.sh:993: wait_for_clean:  get_num_active_clean
       .../qa/workunits/ceph-helpers.sh:863: get_num_active_clean:  local 'expression=('
       .../qa/workunits/ceph-helpers.sh:864: get_num_active_clean:  expression+='contains(.,'\''active'\'') and '
       .../qa/workunits/ceph-helpers.sh:865: get_num_active_clean:  expression+='contains(.,'\''clean'\'') and '
       .../qa/workunits/ceph-helpers.sh:866: get_num_active_clean:  expression+='not(contains(.,'\''stale'\''))'
       .../qa/workunits/ceph-helpers.sh:867: get_num_active_clean:  expression+=')'
       .../qa/workunits/ceph-helpers.sh:871: get_num_active_clean:  ceph --format xml pg dump pgs
       .../qa/workunits/ceph-helpers.sh:872: get_num_active_clean:  xmlstarlet sel -t -m '//pg_stat/state[(contains(.,'\''active'\'') and contains(.,'\''clean'\'') and not(contains(.,'\''stale'\'')))]' -v . -n
       .../qa/workunits/ceph-helpers.sh:873: get_num_active_clean:  grep -v '^$'
       .../qa/workunits/ceph-helpers.sh:873: get_num_active_clean:  wc -l
       ../qa/workunits/ceph-helpers.sh:993: wait_for_clean:  cur_active_clean=0
       ../qa/workunits/ceph-helpers.sh:994: wait_for_clean:  test 0 '!=' 0
       ../qa/workunits/ceph-helpers.sh:998: wait_for_clean:  sleep 1
       .../qa/workunits/ceph-helpers.sh:999: wait_for_clean:  expr 32 + 1
       ../qa/workunits/ceph-helpers.sh:999: wait_for_clean:  timer=33
       ../qa/workunits/ceph-helpers.sh:985: wait_for_clean:  is_clean
       .../qa/workunits/ceph-helpers.sh:952: is_clean:  get_num_pgs
       .../qa/workunits/ceph-helpers.sh:898: get_num_pgs:  ceph --format xml status
       .../qa/workunits/ceph-helpers.sh:899: get_num_pgs:  xmlstarlet sel -t -m //pgmap/num_pgs -v .
       ../qa/workunits/ceph-helpers.sh:952: is_clean:  num_pgs=4
       ../qa/workunits/ceph-helpers.sh:953: is_clean:  test 4 '!=' 0
       .../qa/workunits/ceph-helpers.sh:954: is_clean:  get_num_active_clean
       .../qa/workunits/ceph-helpers.sh:863: get_num_active_clean:  local 'expression=('
       .../qa/workunits/ceph-helpers.sh:864: get_num_active_clean:  expression+='contains(.,'\''active'\'') and '
       .../qa/workunits/ceph-helpers.sh:865: get_num_active_clean:  expression+='contains(.,'\''clean'\'') and '
       .../qa/workunits/ceph-helpers.sh:866: get_num_active_clean:  expression+='not(contains(.,'\''stale'\''))'
       .../qa/workunits/ceph-helpers.sh:867: get_num_active_clean:  expression+=')'
       .../qa/workunits/ceph-helpers.sh:871: get_num_active_clean:  ceph --format xml pg dump pgs
       .../qa/workunits/ceph-helpers.sh:872: get_num_active_clean:  xmlstarlet sel -t -m '//pg_stat/state[(contains(.,'\''active'\'') and contains(.,'\''clean'\'') and not(contains(.,'\''stale'\'')))]' -v . -n
       .../qa/workunits/ceph-helpers.sh:873: get_num_active_clean:  grep -v '^$'
       .../qa/workunits/ceph-helpers.sh:873: get_num_active_clean:  wc -l
       ../qa/workunits/ceph-helpers.sh:954: is_clean:  test 0 = 4
       ../qa/workunits/ceph-helpers.sh:954: is_clean:  return 1
       ../qa/workunits/ceph-helpers.sh:986: wait_for_clean:  get_is_making_recovery_progress
       .../qa/workunits/ceph-helpers.sh:839: get_is_making_recovery_progress:  ceph --format xml status
       .../qa/workunits/ceph-helpers.sh:839: get_is_making_recovery_progress:  xmlstarlet sel -t -m //pgmap/recovering_keys_per_sec -v . -o ' ' -t -m //pgmap/recovering_bytes_per_sec -v . -o ' ' -t -m //pgmap/recovering_objects_per_sec -v .
       ../qa/workunits/ceph-helpers.sh:839: get_is_making_recovery_progress:  local progress=
       ../qa/workunits/ceph-helpers.sh:840: get_is_making_recovery_progress:  test -n ''
       ../qa/workunits/ceph-helpers.sh:988: wait_for_clean:  ((  timer >= 120  ))
       .../qa/workunits/ceph-helpers.sh:993: wait_for_clean:  get_num_active_clean
       .../qa/workunits/ceph-helpers.sh:863: get_num_active_clean:  local 'expression=('
       .../qa/workunits/ceph-helpers.sh:864: get_num_active_clean:  expression+='contains(.,'\''active'\'') and '
       .../qa/workunits/ceph-helpers.sh:865: get_num_active_clean:  expression+='contains(.,'\''clean'\'') and '
       .../qa/workunits/ceph-helpers.sh:866: get_num_active_clean:  expression+='not(contains(.,'\''stale'\''))'
       .../qa/workunits/ceph-helpers.sh:867: get_num_active_clean:  expression+=')'
       .../qa/workunits/ceph-helpers.sh:871: get_num_active_clean:  ceph --format xml pg dump pgs
       .../qa/workunits/ceph-helpers.sh:872: get_num_active_clean:  xmlstarlet sel -t -m '//pg_stat/state[(contains(.,'\''active'\'') and contains(.,'\''clean'\'') and not(contains(.,'\''stale'\'')))]' -v . -n
       .../qa/workunits/ceph-helpers.sh:873: get_num_active_clean:  grep -v '^$'
       .../qa/workunits/ceph-helpers.sh:873: get_num_active_clean:  wc -l
       ../qa/workunits/ceph-helpers.sh:993: wait_for_clean:  cur_active_clean=0
       ../qa/workunits/ceph-helpers.sh:994: wait_for_clean:  test 0 '!=' 0
       ../qa/workunits/ceph-helpers.sh:998: wait_for_clean:  sleep 1
       .../qa/workunits/ceph-helpers.sh:999: wait_for_clean:  expr 33 + 1
       ../qa/workunits/ceph-helpers.sh:999: wait_for_clean:  timer=34
       ../qa/workunits/ceph-helpers.sh:985: wait_for_clean:  is_clean
       .../qa/workunits/ceph-helpers.sh:952: is_clean:  get_num_pgs
       .../qa/workunits/ceph-helpers.sh:898: get_num_pgs:  ceph --format xml status
       .../qa/workunits/ceph-helpers.sh:899: get_num_pgs:  xmlstarlet sel -t -m //pgmap/num_pgs -v .
       ../qa/workunits/ceph-helpers.sh:952: is_clean:  num_pgs=4
       ../qa/workunits/ceph-helpers.sh:953: is_clean:  test 4 '!=' 0
       .../qa/workunits/ceph-helpers.sh:954: is_clean:  get_num_active_clean
       .../qa/workunits/ceph-helpers.sh:863: get_num_active_clean:  local 'expression=('
       .../qa/workunits/ceph-helpers.sh:864: get_num_active_clean:  expression+='contains(.,'\''active'\'') and '
       .../qa/workunits/ceph-helpers.sh:865: get_num_active_clean:  expression+='contains(.,'\''clean'\'') and '
       .../qa/workunits/ceph-helpers.sh:866: get_num_active_clean:  expression+='not(contains(.,'\''stale'\''))'
       .../qa/workunits/ceph-helpers.sh:867: get_num_active_clean:  expression+=')'
       .../qa/workunits/ceph-helpers.sh:871: get_num_active_clean:  ceph --format xml pg dump pgs
       .../qa/workunits/ceph-helpers.sh:872: get_num_active_clean:  xmlstarlet sel -t -m '//pg_stat/state[(contains(.,'\''active'\'') and contains(.,'\''clean'\'') and not(contains(.,'\''stale'\'')))]' -v . -n
       .../qa/workunits/ceph-helpers.sh:873: get_num_active_clean:  grep -v '^$'
       .../qa/workunits/ceph-helpers.sh:873: get_num_active_clean:  wc -l
       ../qa/workunits/ceph-helpers.sh:954: is_clean:  test 0 = 4
       ../qa/workunits/ceph-helpers.sh:954: is_clean:  return 1
       ../qa/workunits/ceph-helpers.sh:986: wait_for_clean:  get_is_making_recovery_progress
       .../qa/workunits/ceph-helpers.sh:839: get_is_making_recovery_progress:  ceph --format xml status
       .../qa/workunits/ceph-helpers.sh:839: get_is_making_recovery_progress:  xmlstarlet sel -t -m //pgmap/recovering_keys_per_sec -v . -o ' ' -t -m //pgmap/recovering_bytes_per_sec -v . -o ' ' -t -m //pgmap/recovering_objects_per_sec -v .
       ../qa/workunits/ceph-helpers.sh:839: get_is_making_recovery_progress:  local progress=
       ../qa/workunits/ceph-helpers.sh:840: get_is_making_recovery_progress:  test -n ''
       ../qa/workunits/ceph-helpers.sh:988: wait_for_clean:  ((  timer >= 120  ))
       .../qa/workunits/ceph-helpers.sh:993: wait_for_clean:  get_num_active_clean
       .../qa/workunits/ceph-helpers.sh:863: get_num_active_clean:  local 'expression=('
       .../qa/workunits/ceph-helpers.sh:864: get_num_active_clean:  expression+='contains(.,'\''active'\'') and '
       .../qa/workunits/ceph-helpers.sh:865: get_num_active_clean:  expression+='contains(.,'\''clean'\'') and '
       .../qa/workunits/ceph-helpers.sh:866: get_num_active_clean:  expression+='not(contains(.,'\''stale'\''))'
       .../qa/workunits/ceph-helpers.sh:867: get_num_active_clean:  expression+=')'
       .../qa/workunits/ceph-helpers.sh:871: get_num_active_clean:  ceph --format xml pg dump pgs
       .../qa/workunits/ceph-helpers.sh:872: get_num_active_clean:  xmlstarlet sel -t -m '//pg_stat/state[(contains(.,'\''active'\'') and contains(.,'\''clean'\'') and not(contains(.,'\''stale'\'')))]' -v . -n
       .../qa/workunits/ceph-helpers.sh:873: get_num_active_clean:  grep -v '^$'
       .../qa/workunits/ceph-helpers.sh:873: get_num_active_clean:  wc -l
       ../qa/workunits/ceph-helpers.sh:993: wait_for_clean:  cur_active_clean=0
       ../qa/workunits/ceph-helpers.sh:994: wait_for_clean:  test 0 '!=' 0
       ../qa/workunits/ceph-helpers.sh:998: wait_for_clean:  sleep 1
       .../qa/workunits/ceph-helpers.sh:999: wait_for_clean:  expr 34 + 1
       ../qa/workunits/ceph-helpers.sh:999: wait_for_clean:  timer=35
       ../qa/workunits/ceph-helpers.sh:985: wait_for_clean:  is_clean
       .../qa/workunits/ceph-helpers.sh:952: is_clean:  get_num_pgs
       .../qa/workunits/ceph-helpers.sh:898: get_num_pgs:  ceph --format xml status
       .../qa/workunits/ceph-helpers.sh:899: get_num_pgs:  xmlstarlet sel -t -m //pgmap/num_pgs -v .
       ../qa/workunits/ceph-helpers.sh:952: is_clean:  num_pgs=4
       ../qa/workunits/ceph-helpers.sh:953: is_clean:  test 4 '!=' 0
       .../qa/workunits/ceph-helpers.sh:954: is_clean:  get_num_active_clean
       .../qa/workunits/ceph-helpers.sh:863: get_num_active_clean:  local 'expression=('
       .../qa/workunits/ceph-helpers.sh:864: get_num_active_clean:  expression+='contains(.,'\''active'\'') and '
       .../qa/workunits/ceph-helpers.sh:865: get_num_active_clean:  expression+='contains(.,'\''clean'\'') and '
       .../qa/workunits/ceph-helpers.sh:866: get_num_active_clean:  expression+='not(contains(.,'\''stale'\''))'
       .../qa/workunits/ceph-helpers.sh:867: get_num_active_clean:  expression+=')'
       .../qa/workunits/ceph-helpers.sh:871: get_num_active_clean:  ceph --format xml pg dump pgs
       .../qa/workunits/ceph-helpers.sh:872: get_num_active_clean:  xmlstarlet sel -t -m '//pg_stat/state[(contains(.,'\''active'\'') and contains(.,'\''clean'\'') and not(contains(.,'\''stale'\'')))]' -v . -n
       .../qa/workunits/ceph-helpers.sh:873: get_num_active_clean:  wc -l
       .../qa/workunits/ceph-helpers.sh:873: get_num_active_clean:  grep -v '^$'
       ../qa/workunits/ceph-helpers.sh:954: is_clean:  test 0 = 4
       ../qa/workunits/ceph-helpers.sh:954: is_clean:  return 1
       ../qa/workunits/ceph-helpers.sh:986: wait_for_clean:  get_is_making_recovery_progress
       .../qa/workunits/ceph-helpers.sh:839: get_is_making_recovery_progress:  ceph --format xml status
       .../qa/workunits/ceph-helpers.sh:839: get_is_making_recovery_progress:  xmlstarlet sel -t -m //pgmap/recovering_keys_per_sec -v . -o ' ' -t -m //pgmap/recovering_bytes_per_sec -v . -o ' ' -t -m //pgmap/recovering_objects_per_sec -v .
       ../qa/workunits/ceph-helpers.sh:839: get_is_making_recovery_progress:  local progress=
       ../qa/workunits/ceph-helpers.sh:840: get_is_making_recovery_progress:  test -n ''
       ../qa/workunits/ceph-helpers.sh:988: wait_for_clean:  ((  timer >= 120  ))
       .../qa/workunits/ceph-helpers.sh:993: wait_for_clean:  get_num_active_clean
       .../qa/workunits/ceph-helpers.sh:863: get_num_active_clean:  local 'expression=('
       .../qa/workunits/ceph-helpers.sh:864: get_num_active_clean:  expression+='contains(.,'\''active'\'') and '
       .../qa/workunits/ceph-helpers.sh:865: get_num_active_clean:  expression+='contains(.,'\''clean'\'') and '
       .../qa/workunits/ceph-helpers.sh:866: get_num_active_clean:  expression+='not(contains(.,'\''stale'\''))'
       .../qa/workunits/ceph-helpers.sh:867: get_num_active_clean:  expression+=')'
       .../qa/workunits/ceph-helpers.sh:871: get_num_active_clean:  ceph --format xml pg dump pgs
       .../qa/workunits/ceph-helpers.sh:872: get_num_active_clean:  xmlstarlet sel -t -m '//pg_stat/state[(contains(.,'\''active'\'') and contains(.,'\''clean'\'') and not(contains(.,'\''stale'\'')))]' -v . -n
       .../qa/workunits/ceph-helpers.sh:873: get_num_active_clean:  grep -v '^$'
       .../qa/workunits/ceph-helpers.sh:873: get_num_active_clean:  wc -l
       ../qa/workunits/ceph-helpers.sh:993: wait_for_clean:  cur_active_clean=0
       ../qa/workunits/ceph-helpers.sh:994: wait_for_clean:  test 0 '!=' 0
       ../qa/workunits/ceph-helpers.sh:998: wait_for_clean:  sleep 1
       .../qa/workunits/ceph-helpers.sh:999: wait_for_clean:  expr 35 + 1
       ../qa/workunits/ceph-helpers.sh:999: wait_for_clean:  timer=36
       ../qa/workunits/ceph-helpers.sh:985: wait_for_clean:  is_clean
       .../qa/workunits/ceph-helpers.sh:952: is_clean:  get_num_pgs
       .../qa/workunits/ceph-helpers.sh:898: get_num_pgs:  ceph --format xml status
       .../qa/workunits/ceph-helpers.sh:899: get_num_pgs:  xmlstarlet sel -t -m //pgmap/num_pgs -v .
       ../qa/workunits/ceph-helpers.sh:952: is_clean:  num_pgs=4
       ../qa/workunits/ceph-helpers.sh:953: is_clean:  test 4 '!=' 0
       .../qa/workunits/ceph-helpers.sh:954: is_clean:  get_num_active_clean
       .../qa/workunits/ceph-helpers.sh:863: get_num_active_clean:  local 'expression=('
       .../qa/workunits/ceph-helpers.sh:864: get_num_active_clean:  expression+='contains(.,'\''active'\'') and '
       .../qa/workunits/ceph-helpers.sh:865: get_num_active_clean:  expression+='contains(.,'\''clean'\'') and '
       .../qa/workunits/ceph-helpers.sh:866: get_num_active_clean:  expression+='not(contains(.,'\''stale'\''))'
       .../qa/workunits/ceph-helpers.sh:867: get_num_active_clean:  expression+=')'
       .../qa/workunits/ceph-helpers.sh:871: get_num_active_clean:  ceph --format xml pg dump pgs
       .../qa/workunits/ceph-helpers.sh:872: get_num_active_clean:  xmlstarlet sel -t -m '//pg_stat/state[(contains(.,'\''active'\'') and contains(.,'\''clean'\'') and not(contains(.,'\''stale'\'')))]' -v . -n
       .../qa/workunits/ceph-helpers.sh:873: get_num_active_clean:  grep -v '^$'
       .../qa/workunits/ceph-helpers.sh:873: get_num_active_clean:  wc -l
       ../qa/workunits/ceph-helpers.sh:954: is_clean:  test 0 = 4
       ../qa/workunits/ceph-helpers.sh:954: is_clean:  return 1
       ../qa/workunits/ceph-helpers.sh:986: wait_for_clean:  get_is_making_recovery_progress
       .../qa/workunits/ceph-helpers.sh:839: get_is_making_recovery_progress:  ceph --format xml status
       .../qa/workunits/ceph-helpers.sh:839: get_is_making_recovery_progress:  xmlstarlet sel -t -m //pgmap/recovering_keys_per_sec -v . -o ' ' -t -m //pgmap/recovering_bytes_per_sec -v . -o ' ' -t -m //pgmap/recovering_objects_per_sec -v .
       ../qa/workunits/ceph-helpers.sh:839: get_is_making_recovery_progress:  local progress=
       ../qa/workunits/ceph-helpers.sh:840: get_is_making_recovery_progress:  test -n ''
       ../qa/workunits/ceph-helpers.sh:988: wait_for_clean:  ((  timer >= 120  ))
       .../qa/workunits/ceph-helpers.sh:993: wait_for_clean:  get_num_active_clean
       .../qa/workunits/ceph-helpers.sh:863: get_num_active_clean:  local 'expression=('
       .../qa/workunits/ceph-helpers.sh:864: get_num_active_clean:  expression+='contains(.,'\''active'\'') and '
       .../qa/workunits/ceph-helpers.sh:865: get_num_active_clean:  expression+='contains(.,'\''clean'\'') and '
       .../qa/workunits/ceph-helpers.sh:866: get_num_active_clean:  expression+='not(contains(.,'\''stale'\''))'
       .../qa/workunits/ceph-helpers.sh:867: get_num_active_clean:  expression+=')'
       .../qa/workunits/ceph-helpers.sh:871: get_num_active_clean:  ceph --format xml pg dump pgs
       .../qa/workunits/ceph-helpers.sh:872: get_num_active_clean:  xmlstarlet sel -t -m '//pg_stat/state[(contains(.,'\''active'\'') and contains(.,'\''clean'\'') and not(contains(.,'\''stale'\'')))]' -v . -n
       .../qa/workunits/ceph-helpers.sh:873: get_num_active_clean:  grep -v '^$'
       .../qa/workunits/ceph-helpers.sh:873: get_num_active_clean:  wc -l
       ../qa/workunits/ceph-helpers.sh:993: wait_for_clean:  cur_active_clean=0
       ../qa/workunits/ceph-helpers.sh:994: wait_for_clean:  test 0 '!=' 0
       ../qa/workunits/ceph-helpers.sh:998: wait_for_clean:  sleep 1
       .../qa/workunits/ceph-helpers.sh:999: wait_for_clean:  expr 36 + 1
       ../qa/workunits/ceph-helpers.sh:999: wait_for_clean:  timer=37
       ../qa/workunits/ceph-helpers.sh:985: wait_for_clean:  is_clean
       .../qa/workunits/ceph-helpers.sh:952: is_clean:  get_num_pgs
       .../qa/workunits/ceph-helpers.sh:898: get_num_pgs:  ceph --format xml status
       .../qa/workunits/ceph-helpers.sh:899: get_num_pgs:  xmlstarlet sel -t -m //pgmap/num_pgs -v .
       ../qa/workunits/ceph-helpers.sh:952: is_clean:  num_pgs=4
       ../qa/workunits/ceph-helpers.sh:953: is_clean:  test 4 '!=' 0
       .../qa/workunits/ceph-helpers.sh:954: is_clean:  get_num_active_clean
       .../qa/workunits/ceph-helpers.sh:863: get_num_active_clean:  local 'expression=('
       .../qa/workunits/ceph-helpers.sh:864: get_num_active_clean:  expression+='contains(.,'\''active'\'') and '
       .../qa/workunits/ceph-helpers.sh:865: get_num_active_clean:  expression+='contains(.,'\''clean'\'') and '
       .../qa/workunits/ceph-helpers.sh:866: get_num_active_clean:  expression+='not(contains(.,'\''stale'\''))'
       .../qa/workunits/ceph-helpers.sh:867: get_num_active_clean:  expression+=')'
       .../qa/workunits/ceph-helpers.sh:871: get_num_active_clean:  ceph --format xml pg dump pgs
       .../qa/workunits/ceph-helpers.sh:872: get_num_active_clean:  xmlstarlet sel -t -m '//pg_stat/state[(contains(.,'\''active'\'') and contains(.,'\''clean'\'') and not(contains(.,'\''stale'\'')))]' -v . -n
       .../qa/workunits/ceph-helpers.sh:873: get_num_active_clean:  grep -v '^$'
       .../qa/workunits/ceph-helpers.sh:873: get_num_active_clean:  wc -l
       ../qa/workunits/ceph-helpers.sh:954: is_clean:  test 0 = 4
       ../qa/workunits/ceph-helpers.sh:954: is_clean:  return 1
       ../qa/workunits/ceph-helpers.sh:986: wait_for_clean:  get_is_making_recovery_progress
       .../qa/workunits/ceph-helpers.sh:839: get_is_making_recovery_progress:  ceph --format xml status
       .../qa/workunits/ceph-helpers.sh:839: get_is_making_recovery_progress:  xmlstarlet sel -t -m //pgmap/recovering_keys_per_sec -v . -o ' ' -t -m //pgmap/recovering_bytes_per_sec -v . -o ' ' -t -m //pgmap/recovering_objects_per_sec -v .
       ../qa/workunits/ceph-helpers.sh:839: get_is_making_recovery_progress:  local progress=
       ../qa/workunits/ceph-helpers.sh:840: get_is_making_recovery_progress:  test -n ''
       ../qa/workunits/ceph-helpers.sh:988: wait_for_clean:  ((  timer >= 120  ))
       .../qa/workunits/ceph-helpers.sh:993: wait_for_clean:  get_num_active_clean
       .../qa/workunits/ceph-helpers.sh:863: get_num_active_clean:  local 'expression=('
       .../qa/workunits/ceph-helpers.sh:864: get_num_active_clean:  expression+='contains(.,'\''active'\'') and '
       .../qa/workunits/ceph-helpers.sh:865: get_num_active_clean:  expression+='contains(.,'\''clean'\'') and '
       .../qa/workunits/ceph-helpers.sh:866: get_num_active_clean:  expression+='not(contains(.,'\''stale'\''))'
       .../qa/workunits/ceph-helpers.sh:867: get_num_active_clean:  expression+=')'
       .../qa/workunits/ceph-helpers.sh:871: get_num_active_clean:  ceph --format xml pg dump pgs
       .../qa/workunits/ceph-helpers.sh:872: get_num_active_clean:  xmlstarlet sel -t -m '//pg_stat/state[(contains(.,'\''active'\'') and contains(.,'\''clean'\'') and not(contains(.,'\''stale'\'')))]' -v . -n
       .../qa/workunits/ceph-helpers.sh:873: get_num_active_clean:  grep -v '^$'
       .../qa/workunits/ceph-helpers.sh:873: get_num_active_clean:  wc -l
       ../qa/workunits/ceph-helpers.sh:993: wait_for_clean:  cur_active_clean=0
       ../qa/workunits/ceph-helpers.sh:994: wait_for_clean:  test 0 '!=' 0
       ../qa/workunits/ceph-helpers.sh:998: wait_for_clean:  sleep 1
       .../qa/workunits/ceph-helpers.sh:999: wait_for_clean:  expr 37 + 1
       ../qa/workunits/ceph-helpers.sh:999: wait_for_clean:  timer=38
       ../qa/workunits/ceph-helpers.sh:985: wait_for_clean:  is_clean
       .../qa/workunits/ceph-helpers.sh:952: is_clean:  get_num_pgs
       .../qa/workunits/ceph-helpers.sh:898: get_num_pgs:  ceph --format xml status
       .../qa/workunits/ceph-helpers.sh:899: get_num_pgs:  xmlstarlet sel -t -m //pgmap/num_pgs -v .
       ../qa/workunits/ceph-helpers.sh:952: is_clean:  num_pgs=4
       ../qa/workunits/ceph-helpers.sh:953: is_clean:  test 4 '!=' 0
       .../qa/workunits/ceph-helpers.sh:954: is_clean:  get_num_active_clean
       .../qa/workunits/ceph-helpers.sh:863: get_num_active_clean:  local 'expression=('
       .../qa/workunits/ceph-helpers.sh:864: get_num_active_clean:  expression+='contains(.,'\''active'\'') and '
       .../qa/workunits/ceph-helpers.sh:865: get_num_active_clean:  expression+='contains(.,'\''clean'\'') and '
       .../qa/workunits/ceph-helpers.sh:866: get_num_active_clean:  expression+='not(contains(.,'\''stale'\''))'
       .../qa/workunits/ceph-helpers.sh:867: get_num_active_clean:  expression+=')'
       .../qa/workunits/ceph-helpers.sh:871: get_num_active_clean:  ceph --format xml pg dump pgs
       .../qa/workunits/ceph-helpers.sh:872: get_num_active_clean:  xmlstarlet sel -t -m '//pg_stat/state[(contains(.,'\''active'\'') and contains(.,'\''clean'\'') and not(contains(.,'\''stale'\'')))]' -v . -n
       .../qa/workunits/ceph-helpers.sh:873: get_num_active_clean:  grep -v '^$'
       .../qa/workunits/ceph-helpers.sh:873: get_num_active_clean:  wc -l
       ../qa/workunits/ceph-helpers.sh:954: is_clean:  test 0 = 4
       ../qa/workunits/ceph-helpers.sh:954: is_clean:  return 1
       ../qa/workunits/ceph-helpers.sh:986: wait_for_clean:  get_is_making_recovery_progress
       .../qa/workunits/ceph-helpers.sh:839: get_is_making_recovery_progress:  ceph --format xml status
       .../qa/workunits/ceph-helpers.sh:839: get_is_making_recovery_progress:  xmlstarlet sel -t -m //pgmap/recovering_keys_per_sec -v . -o ' ' -t -m //pgmap/recovering_bytes_per_sec -v . -o ' ' -t -m //pgmap/recovering_objects_per_sec -v .
       ../qa/workunits/ceph-helpers.sh:839: get_is_making_recovery_progress:  local progress=
       ../qa/workunits/ceph-helpers.sh:840: get_is_making_recovery_progress:  test -n ''
       ../qa/workunits/ceph-helpers.sh:988: wait_for_clean:  ((  timer >= 120  ))
       .../qa/workunits/ceph-helpers.sh:993: wait_for_clean:  get_num_active_clean
       .../qa/workunits/ceph-helpers.sh:863: get_num_active_clean:  local 'expression=('
       .../qa/workunits/ceph-helpers.sh:864: get_num_active_clean:  expression+='contains(.,'\''active'\'') and '
       .../qa/workunits/ceph-helpers.sh:865: get_num_active_clean:  expression+='contains(.,'\''clean'\'') and '
       .../qa/workunits/ceph-helpers.sh:866: get_num_active_clean:  expression+='not(contains(.,'\''stale'\''))'
       .../qa/workunits/ceph-helpers.sh:867: get_num_active_clean:  expression+=')'
       .../qa/workunits/ceph-helpers.sh:871: get_num_active_clean:  ceph --format xml pg dump pgs
       .../qa/workunits/ceph-helpers.sh:872: get_num_active_clean:  xmlstarlet sel -t -m '//pg_stat/state[(contains(.,'\''active'\'') and contains(.,'\''clean'\'') and not(contains(.,'\''stale'\'')))]' -v . -n
       .../qa/workunits/ceph-helpers.sh:873: get_num_active_clean:  grep -v '^$'
       .../qa/workunits/ceph-helpers.sh:873: get_num_active_clean:  wc -l
       ../qa/workunits/ceph-helpers.sh:993: wait_for_clean:  cur_active_clean=0
       ../qa/workunits/ceph-helpers.sh:994: wait_for_clean:  test 0 '!=' 0
       ../qa/workunits/ceph-helpers.sh:998: wait_for_clean:  sleep 1
       .../qa/workunits/ceph-helpers.sh:999: wait_for_clean:  expr 38 + 1
       ../qa/workunits/ceph-helpers.sh:999: wait_for_clean:  timer=39
       ../qa/workunits/ceph-helpers.sh:985: wait_for_clean:  is_clean
       .../qa/workunits/ceph-helpers.sh:952: is_clean:  get_num_pgs
       .../qa/workunits/ceph-helpers.sh:898: get_num_pgs:  ceph --format xml status
       .../qa/workunits/ceph-helpers.sh:899: get_num_pgs:  xmlstarlet sel -t -m //pgmap/num_pgs -v .
       ../qa/workunits/ceph-helpers.sh:952: is_clean:  num_pgs=4
       ../qa/workunits/ceph-helpers.sh:953: is_clean:  test 4 '!=' 0
       .../qa/workunits/ceph-helpers.sh:954: is_clean:  get_num_active_clean
       .../qa/workunits/ceph-helpers.sh:863: get_num_active_clean:  local 'expression=('
       .../qa/workunits/ceph-helpers.sh:864: get_num_active_clean:  expression+='contains(.,'\''active'\'') and '
       .../qa/workunits/ceph-helpers.sh:865: get_num_active_clean:  expression+='contains(.,'\''clean'\'') and '
       .../qa/workunits/ceph-helpers.sh:866: get_num_active_clean:  expression+='not(contains(.,'\''stale'\''))'
       .../qa/workunits/ceph-helpers.sh:867: get_num_active_clean:  expression+=')'
       .../qa/workunits/ceph-helpers.sh:871: get_num_active_clean:  ceph --format xml pg dump pgs
       .../qa/workunits/ceph-helpers.sh:872: get_num_active_clean:  xmlstarlet sel -t -m '//pg_stat/state[(contains(.,'\''active'\'') and contains(.,'\''clean'\'') and not(contains(.,'\''stale'\'')))]' -v . -n
       .../qa/workunits/ceph-helpers.sh:873: get_num_active_clean:  wc -l
       .../qa/workunits/ceph-helpers.sh:873: get_num_active_clean:  grep -v '^$'
       ../qa/workunits/ceph-helpers.sh:954: is_clean:  test 0 = 4
       ../qa/workunits/ceph-helpers.sh:954: is_clean:  return 1
       ../qa/workunits/ceph-helpers.sh:986: wait_for_clean:  get_is_making_recovery_progress
       .../qa/workunits/ceph-helpers.sh:839: get_is_making_recovery_progress:  ceph --format xml status
       .../qa/workunits/ceph-helpers.sh:839: get_is_making_recovery_progress:  xmlstarlet sel -t -m //pgmap/recovering_keys_per_sec -v . -o ' ' -t -m //pgmap/recovering_bytes_per_sec -v . -o ' ' -t -m //pgmap/recovering_objects_per_sec -v .
       ../qa/workunits/ceph-helpers.sh:839: get_is_making_recovery_progress:  local progress=
       ../qa/workunits/ceph-helpers.sh:840: get_is_making_recovery_progress:  test -n ''
       ../qa/workunits/ceph-helpers.sh:988: wait_for_clean:  ((  timer >= 120  ))
       .../qa/workunits/ceph-helpers.sh:993: wait_for_clean:  get_num_active_clean
       .../qa/workunits/ceph-helpers.sh:863: get_num_active_clean:  local 'expression=('
       .../qa/workunits/ceph-helpers.sh:864: get_num_active_clean:  expression+='contains(.,'\''active'\'') and '
       .../qa/workunits/ceph-helpers.sh:865: get_num_active_clean:  expression+='contains(.,'\''clean'\'') and '
       .../qa/workunits/ceph-helpers.sh:866: get_num_active_clean:  expression+='not(contains(.,'\''stale'\''))'
       .../qa/workunits/ceph-helpers.sh:867: get_num_active_clean:  expression+=')'
       .../qa/workunits/ceph-helpers.sh:871: get_num_active_clean:  ceph --format xml pg dump pgs
       .../qa/workunits/ceph-helpers.sh:872: get_num_active_clean:  xmlstarlet sel -t -m '//pg_stat/state[(contains(.,'\''active'\'') and contains(.,'\''clean'\'') and not(contains(.,'\''stale'\'')))]' -v . -n
       .../qa/workunits/ceph-helpers.sh:873: get_num_active_clean:  wc -l
       .../qa/workunits/ceph-helpers.sh:873: get_num_active_clean:  grep -v '^$'
       ../qa/workunits/ceph-helpers.sh:993: wait_for_clean:  cur_active_clean=0
       ../qa/workunits/ceph-helpers.sh:994: wait_for_clean:  test 0 '!=' 0
       ../qa/workunits/ceph-helpers.sh:998: wait_for_clean:  sleep 1
       .../qa/workunits/ceph-helpers.sh:999: wait_for_clean:  expr 39 + 1
       ../qa/workunits/ceph-helpers.sh:999: wait_for_clean:  timer=40
       ../qa/workunits/ceph-helpers.sh:985: wait_for_clean:  is_clean
       .../qa/workunits/ceph-helpers.sh:952: is_clean:  get_num_pgs
       .../qa/workunits/ceph-helpers.sh:898: get_num_pgs:  ceph --format xml status
       .../qa/workunits/ceph-helpers.sh:899: get_num_pgs:  xmlstarlet sel -t -m //pgmap/num_pgs -v .
       ../qa/workunits/ceph-helpers.sh:952: is_clean:  num_pgs=4
       ../qa/workunits/ceph-helpers.sh:953: is_clean:  test 4 '!=' 0
       .../qa/workunits/ceph-helpers.sh:954: is_clean:  get_num_active_clean
       .../qa/workunits/ceph-helpers.sh:863: get_num_active_clean:  local 'expression=('
       .../qa/workunits/ceph-helpers.sh:864: get_num_active_clean:  expression+='contains(.,'\''active'\'') and '
       .../qa/workunits/ceph-helpers.sh:865: get_num_active_clean:  expression+='contains(.,'\''clean'\'') and '
       .../qa/workunits/ceph-helpers.sh:866: get_num_active_clean:  expression+='not(contains(.,'\''stale'\''))'
       .../qa/workunits/ceph-helpers.sh:867: get_num_active_clean:  expression+=')'
       .../qa/workunits/ceph-helpers.sh:871: get_num_active_clean:  ceph --format xml pg dump pgs
       .../qa/workunits/ceph-helpers.sh:872: get_num_active_clean:  xmlstarlet sel -t -m '//pg_stat/state[(contains(.,'\''active'\'') and contains(.,'\''clean'\'') and not(contains(.,'\''stale'\'')))]' -v . -n
       .../qa/workunits/ceph-helpers.sh:873: get_num_active_clean:  grep -v '^$'
       .../qa/workunits/ceph-helpers.sh:873: get_num_active_clean:  wc -l
       ../qa/workunits/ceph-helpers.sh:954: is_clean:  test 0 = 4
       ../qa/workunits/ceph-helpers.sh:954: is_clean:  return 1
       ../qa/workunits/ceph-helpers.sh:986: wait_for_clean:  get_is_making_recovery_progress
       .../qa/workunits/ceph-helpers.sh:839: get_is_making_recovery_progress:  ceph --format xml status
       .../qa/workunits/ceph-helpers.sh:839: get_is_making_recovery_progress:  xmlstarlet sel -t -m //pgmap/recovering_keys_per_sec -v . -o ' ' -t -m //pgmap/recovering_bytes_per_sec -v . -o ' ' -t -m //pgmap/recovering_objects_per_sec -v .
       ../qa/workunits/ceph-helpers.sh:839: get_is_making_recovery_progress:  local progress=
       ../qa/workunits/ceph-helpers.sh:840: get_is_making_recovery_progress:  test -n ''
       ../qa/workunits/ceph-helpers.sh:988: wait_for_clean:  ((  timer >= 120  ))
       .../qa/workunits/ceph-helpers.sh:993: wait_for_clean:  get_num_active_clean
       .../qa/workunits/ceph-helpers.sh:863: get_num_active_clean:  local 'expression=('
       .../qa/workunits/ceph-helpers.sh:864: get_num_active_clean:  expression+='contains(.,'\''active'\'') and '
       .../qa/workunits/ceph-helpers.sh:865: get_num_active_clean:  expression+='contains(.,'\''clean'\'') and '
       .../qa/workunits/ceph-helpers.sh:866: get_num_active_clean:  expression+='not(contains(.,'\''stale'\''))'
       .../qa/workunits/ceph-helpers.sh:867: get_num_active_clean:  expression+=')'
       .../qa/workunits/ceph-helpers.sh:871: get_num_active_clean:  ceph --format xml pg dump pgs
       .../qa/workunits/ceph-helpers.sh:872: get_num_active_clean:  xmlstarlet sel -t -m '//pg_stat/state[(contains(.,'\''active'\'') and contains(.,'\''clean'\'') and not(contains(.,'\''stale'\'')))]' -v . -n
       .../qa/workunits/ceph-helpers.sh:873: get_num_active_clean:  grep -v '^$'
       .../qa/workunits/ceph-helpers.sh:873: get_num_active_clean:  wc -l
       ../qa/workunits/ceph-helpers.sh:993: wait_for_clean:  cur_active_clean=0
       ../qa/workunits/ceph-helpers.sh:994: wait_for_clean:  test 0 '!=' 0
       ../qa/workunits/ceph-helpers.sh:998: wait_for_clean:  sleep 1
       .../qa/workunits/ceph-helpers.sh:999: wait_for_clean:  expr 40 + 1
       ../qa/workunits/ceph-helpers.sh:999: wait_for_clean:  timer=41
       ../qa/workunits/ceph-helpers.sh:985: wait_for_clean:  is_clean
       .../qa/workunits/ceph-helpers.sh:952: is_clean:  get_num_pgs
       .../qa/workunits/ceph-helpers.sh:898: get_num_pgs:  ceph --format xml status
       .../qa/workunits/ceph-helpers.sh:899: get_num_pgs:  xmlstarlet sel -t -m //pgmap/num_pgs -v .
       ../qa/workunits/ceph-helpers.sh:952: is_clean:  num_pgs=4
       ../qa/workunits/ceph-helpers.sh:953: is_clean:  test 4 '!=' 0
       .../qa/workunits/ceph-helpers.sh:954: is_clean:  get_num_active_clean
       .../qa/workunits/ceph-helpers.sh:863: get_num_active_clean:  local 'expression=('
       .../qa/workunits/ceph-helpers.sh:864: get_num_active_clean:  expression+='contains(.,'\''active'\'') and '
       .../qa/workunits/ceph-helpers.sh:865: get_num_active_clean:  expression+='contains(.,'\''clean'\'') and '
       .../qa/workunits/ceph-helpers.sh:866: get_num_active_clean:  expression+='not(contains(.,'\''stale'\''))'
       .../qa/workunits/ceph-helpers.sh:867: get_num_active_clean:  expression+=')'
       .../qa/workunits/ceph-helpers.sh:871: get_num_active_clean:  ceph --format xml pg dump pgs
       .../qa/workunits/ceph-helpers.sh:873: get_num_active_clean:  grep -v '^$'
       .../qa/workunits/ceph-helpers.sh:873: get_num_active_clean:  wc -l
       .../qa/workunits/ceph-helpers.sh:872: get_num_active_clean:  xmlstarlet sel -t -m '//pg_stat/state[(contains(.,'\''active'\'') and contains(.,'\''clean'\'') and not(contains(.,'\''stale'\'')))]' -v . -n
       ../qa/workunits/ceph-helpers.sh:954: is_clean:  test 0 = 4
       ../qa/workunits/ceph-helpers.sh:954: is_clean:  return 1
       ../qa/workunits/ceph-helpers.sh:986: wait_for_clean:  get_is_making_recovery_progress
       .../qa/workunits/ceph-helpers.sh:839: get_is_making_recovery_progress:  ceph --format xml status
       .../qa/workunits/ceph-helpers.sh:839: get_is_making_recovery_progress:  xmlstarlet sel -t -m //pgmap/recovering_keys_per_sec -v . -o ' ' -t -m //pgmap/recovering_bytes_per_sec -v . -o ' ' -t -m //pgmap/recovering_objects_per_sec -v .
       ../qa/workunits/ceph-helpers.sh:839: get_is_making_recovery_progress:  local progress=
       ../qa/workunits/ceph-helpers.sh:840: get_is_making_recovery_progress:  test -n ''
       ../qa/workunits/ceph-helpers.sh:988: wait_for_clean:  ((  timer >= 120  ))
       .../qa/workunits/ceph-helpers.sh:993: wait_for_clean:  get_num_active_clean
       .../qa/workunits/ceph-helpers.sh:863: get_num_active_clean:  local 'expression=('
       .../qa/workunits/ceph-helpers.sh:864: get_num_active_clean:  expression+='contains(.,'\''active'\'') and '
       .../qa/workunits/ceph-helpers.sh:865: get_num_active_clean:  expression+='contains(.,'\''clean'\'') and '
       .../qa/workunits/ceph-helpers.sh:866: get_num_active_clean:  expression+='not(contains(.,'\''stale'\''))'
       .../qa/workunits/ceph-helpers.sh:867: get_num_active_clean:  expression+=')'
       .../qa/workunits/ceph-helpers.sh:871: get_num_active_clean:  ceph --format xml pg dump pgs
       .../qa/workunits/ceph-helpers.sh:873: get_num_active_clean:  grep -v '^$'
       .../qa/workunits/ceph-helpers.sh:872: get_num_active_clean:  xmlstarlet sel -t -m '//pg_stat/state[(contains(.,'\''active'\'') and contains(.,'\''clean'\'') and not(contains(.,'\''stale'\'')))]' -v . -n
       .../qa/workunits/ceph-helpers.sh:873: get_num_active_clean:  wc -l
       ../qa/workunits/ceph-helpers.sh:993: wait_for_clean:  cur_active_clean=0
       ../qa/workunits/ceph-helpers.sh:994: wait_for_clean:  test 0 '!=' 0
       ../qa/workunits/ceph-helpers.sh:998: wait_for_clean:  sleep 1
       .../qa/workunits/ceph-helpers.sh:999: wait_for_clean:  expr 41 + 1
       ../qa/workunits/ceph-helpers.sh:999: wait_for_clean:  timer=42
       ../qa/workunits/ceph-helpers.sh:985: wait_for_clean:  is_clean
       .../qa/workunits/ceph-helpers.sh:952: is_clean:  get_num_pgs
       .../qa/workunits/ceph-helpers.sh:898: get_num_pgs:  ceph --format xml status
       .../qa/workunits/ceph-helpers.sh:899: get_num_pgs:  xmlstarlet sel -t -m //pgmap/num_pgs -v .
       ../qa/workunits/ceph-helpers.sh:952: is_clean:  num_pgs=4
       ../qa/workunits/ceph-helpers.sh:953: is_clean:  test 4 '!=' 0
       .../qa/workunits/ceph-helpers.sh:954: is_clean:  get_num_active_clean
       .../qa/workunits/ceph-helpers.sh:863: get_num_active_clean:  local 'expression=('
       .../qa/workunits/ceph-helpers.sh:864: get_num_active_clean:  expression+='contains(.,'\''active'\'') and '
       .../qa/workunits/ceph-helpers.sh:865: get_num_active_clean:  expression+='contains(.,'\''clean'\'') and '
       .../qa/workunits/ceph-helpers.sh:866: get_num_active_clean:  expression+='not(contains(.,'\''stale'\''))'
       .../qa/workunits/ceph-helpers.sh:867: get_num_active_clean:  expression+=')'
       .../qa/workunits/ceph-helpers.sh:871: get_num_active_clean:  ceph --format xml pg dump pgs
       .../qa/workunits/ceph-helpers.sh:872: get_num_active_clean:  xmlstarlet sel -t -m '//pg_stat/state[(contains(.,'\''active'\'') and contains(.,'\''clean'\'') and not(contains(.,'\''stale'\'')))]' -v . -n
       .../qa/workunits/ceph-helpers.sh:873: get_num_active_clean:  grep -v '^$'
       .../qa/workunits/ceph-helpers.sh:873: get_num_active_clean:  wc -l
       ../qa/workunits/ceph-helpers.sh:954: is_clean:  test 0 = 4
       ../qa/workunits/ceph-helpers.sh:954: is_clean:  return 1
       ../qa/workunits/ceph-helpers.sh:986: wait_for_clean:  get_is_making_recovery_progress
       .../qa/workunits/ceph-helpers.sh:839: get_is_making_recovery_progress:  ceph --format xml status
       .../qa/workunits/ceph-helpers.sh:839: get_is_making_recovery_progress:  xmlstarlet sel -t -m //pgmap/recovering_keys_per_sec -v . -o ' ' -t -m //pgmap/recovering_bytes_per_sec -v . -o ' ' -t -m //pgmap/recovering_objects_per_sec -v .
       ../qa/workunits/ceph-helpers.sh:839: get_is_making_recovery_progress:  local progress=
       ../qa/workunits/ceph-helpers.sh:840: get_is_making_recovery_progress:  test -n ''
       ../qa/workunits/ceph-helpers.sh:988: wait_for_clean:  ((  timer >= 120  ))
       .../qa/workunits/ceph-helpers.sh:993: wait_for_clean:  get_num_active_clean
       .../qa/workunits/ceph-helpers.sh:863: get_num_active_clean:  local 'expression=('
       .../qa/workunits/ceph-helpers.sh:864: get_num_active_clean:  expression+='contains(.,'\''active'\'') and '
       .../qa/workunits/ceph-helpers.sh:865: get_num_active_clean:  expression+='contains(.,'\''clean'\'') and '
       .../qa/workunits/ceph-helpers.sh:866: get_num_active_clean:  expression+='not(contains(.,'\''stale'\''))'
       .../qa/workunits/ceph-helpers.sh:867: get_num_active_clean:  expression+=')'
       .../qa/workunits/ceph-helpers.sh:871: get_num_active_clean:  ceph --format xml pg dump pgs
       .../qa/workunits/ceph-helpers.sh:872: get_num_active_clean:  xmlstarlet sel -t -m '//pg_stat/state[(contains(.,'\''active'\'') and contains(.,'\''clean'\'') and not(contains(.,'\''stale'\'')))]' -v . -n
       .../qa/workunits/ceph-helpers.sh:873: get_num_active_clean:  grep -v '^$'
       .../qa/workunits/ceph-helpers.sh:873: get_num_active_clean:  wc -l
       ../qa/workunits/ceph-helpers.sh:993: wait_for_clean:  cur_active_clean=0
       ../qa/workunits/ceph-helpers.sh:994: wait_for_clean:  test 0 '!=' 0
       ../qa/workunits/ceph-helpers.sh:998: wait_for_clean:  sleep 1
       .../qa/workunits/ceph-helpers.sh:999: wait_for_clean:  expr 42 + 1
       ../qa/workunits/ceph-helpers.sh:999: wait_for_clean:  timer=43
       ../qa/workunits/ceph-helpers.sh:985: wait_for_clean:  is_clean
       .../qa/workunits/ceph-helpers.sh:952: is_clean:  get_num_pgs
       .../qa/workunits/ceph-helpers.sh:898: get_num_pgs:  ceph --format xml status
       .../qa/workunits/ceph-helpers.sh:899: get_num_pgs:  xmlstarlet sel -t -m //pgmap/num_pgs -v .
       ../qa/workunits/ceph-helpers.sh:952: is_clean:  num_pgs=4
       ../qa/workunits/ceph-helpers.sh:953: is_clean:  test 4 '!=' 0
       .../qa/workunits/ceph-helpers.sh:954: is_clean:  get_num_active_clean
       .../qa/workunits/ceph-helpers.sh:863: get_num_active_clean:  local 'expression=('
       .../qa/workunits/ceph-helpers.sh:864: get_num_active_clean:  expression+='contains(.,'\''active'\'') and '
       .../qa/workunits/ceph-helpers.sh:865: get_num_active_clean:  expression+='contains(.,'\''clean'\'') and '
       .../qa/workunits/ceph-helpers.sh:866: get_num_active_clean:  expression+='not(contains(.,'\''stale'\''))'
       .../qa/workunits/ceph-helpers.sh:867: get_num_active_clean:  expression+=')'
       .../qa/workunits/ceph-helpers.sh:871: get_num_active_clean:  ceph --format xml pg dump pgs
       .../qa/workunits/ceph-helpers.sh:872: get_num_active_clean:  xmlstarlet sel -t -m '//pg_stat/state[(contains(.,'\''active'\'') and contains(.,'\''clean'\'') and not(contains(.,'\''stale'\'')))]' -v . -n
       .../qa/workunits/ceph-helpers.sh:873: get_num_active_clean:  grep -v '^$'
       .../qa/workunits/ceph-helpers.sh:873: get_num_active_clean:  wc -l
       ../qa/workunits/ceph-helpers.sh:954: is_clean:  test 0 = 4
       ../qa/workunits/ceph-helpers.sh:954: is_clean:  return 1
       ../qa/workunits/ceph-helpers.sh:986: wait_for_clean:  get_is_making_recovery_progress
       .../qa/workunits/ceph-helpers.sh:839: get_is_making_recovery_progress:  ceph --format xml status
       .../qa/workunits/ceph-helpers.sh:839: get_is_making_recovery_progress:  xmlstarlet sel -t -m //pgmap/recovering_keys_per_sec -v . -o ' ' -t -m //pgmap/recovering_bytes_per_sec -v . -o ' ' -t -m //pgmap/recovering_objects_per_sec -v .
       ../qa/workunits/ceph-helpers.sh:839: get_is_making_recovery_progress:  local progress=
       ../qa/workunits/ceph-helpers.sh:840: get_is_making_recovery_progress:  test -n ''
       ../qa/workunits/ceph-helpers.sh:988: wait_for_clean:  ((  timer >= 120  ))
       .../qa/workunits/ceph-helpers.sh:993: wait_for_clean:  get_num_active_clean
       .../qa/workunits/ceph-helpers.sh:863: get_num_active_clean:  local 'expression=('
       .../qa/workunits/ceph-helpers.sh:864: get_num_active_clean:  expression+='contains(.,'\''active'\'') and '
       .../qa/workunits/ceph-helpers.sh:865: get_num_active_clean:  expression+='contains(.,'\''clean'\'') and '
       .../qa/workunits/ceph-helpers.sh:866: get_num_active_clean:  expression+='not(contains(.,'\''stale'\''))'
       .../qa/workunits/ceph-helpers.sh:867: get_num_active_clean:  expression+=')'
       .../qa/workunits/ceph-helpers.sh:871: get_num_active_clean:  ceph --format xml pg dump pgs
       .../qa/workunits/ceph-helpers.sh:872: get_num_active_clean:  xmlstarlet sel -t -m '//pg_stat/state[(contains(.,'\''active'\'') and contains(.,'\''clean'\'') and not(contains(.,'\''stale'\'')))]' -v . -n
       .../qa/workunits/ceph-helpers.sh:873: get_num_active_clean:  grep -v '^$'
       .../qa/workunits/ceph-helpers.sh:873: get_num_active_clean:  wc -l
       ../qa/workunits/ceph-helpers.sh:993: wait_for_clean:  cur_active_clean=0
       ../qa/workunits/ceph-helpers.sh:994: wait_for_clean:  test 0 '!=' 0
       ../qa/workunits/ceph-helpers.sh:998: wait_for_clean:  sleep 1
       .../qa/workunits/ceph-helpers.sh:999: wait_for_clean:  expr 43 + 1
       ../qa/workunits/ceph-helpers.sh:999: wait_for_clean:  timer=44
       ../qa/workunits/ceph-helpers.sh:985: wait_for_clean:  is_clean
       .../qa/workunits/ceph-helpers.sh:952: is_clean:  get_num_pgs
       .../qa/workunits/ceph-helpers.sh:898: get_num_pgs:  ceph --format xml status
       .../qa/workunits/ceph-helpers.sh:899: get_num_pgs:  xmlstarlet sel -t -m //pgmap/num_pgs -v .
       ../qa/workunits/ceph-helpers.sh:952: is_clean:  num_pgs=4
       ../qa/workunits/ceph-helpers.sh:953: is_clean:  test 4 '!=' 0
       .../qa/workunits/ceph-helpers.sh:954: is_clean:  get_num_active_clean
       .../qa/workunits/ceph-helpers.sh:863: get_num_active_clean:  local 'expression=('
       .../qa/workunits/ceph-helpers.sh:864: get_num_active_clean:  expression+='contains(.,'\''active'\'') and '
       .../qa/workunits/ceph-helpers.sh:865: get_num_active_clean:  expression+='contains(.,'\''clean'\'') and '
       .../qa/workunits/ceph-helpers.sh:866: get_num_active_clean:  expression+='not(contains(.,'\''stale'\''))'
       .../qa/workunits/ceph-helpers.sh:867: get_num_active_clean:  expression+=')'
       .../qa/workunits/ceph-helpers.sh:871: get_num_active_clean:  ceph --format xml pg dump pgs
       .../qa/workunits/ceph-helpers.sh:872: get_num_active_clean:  xmlstarlet sel -t -m '//pg_stat/state[(contains(.,'\''active'\'') and contains(.,'\''clean'\'') and not(contains(.,'\''stale'\'')))]' -v . -n
       .../qa/workunits/ceph-helpers.sh:873: get_num_active_clean:  grep -v '^$'
       .../qa/workunits/ceph-helpers.sh:873: get_num_active_clean:  wc -l
       ../qa/workunits/ceph-helpers.sh:954: is_clean:  test 0 = 4
       ../qa/workunits/ceph-helpers.sh:954: is_clean:  return 1
       ../qa/workunits/ceph-helpers.sh:986: wait_for_clean:  get_is_making_recovery_progress
       .../qa/workunits/ceph-helpers.sh:839: get_is_making_recovery_progress:  ceph --format xml status
       .../qa/workunits/ceph-helpers.sh:839: get_is_making_recovery_progress:  xmlstarlet sel -t -m //pgmap/recovering_keys_per_sec -v . -o ' ' -t -m //pgmap/recovering_bytes_per_sec -v . -o ' ' -t -m //pgmap/recovering_objects_per_sec -v .
       ../qa/workunits/ceph-helpers.sh:839: get_is_making_recovery_progress:  local progress=
       ../qa/workunits/ceph-helpers.sh:840: get_is_making_recovery_progress:  test -n ''
       ../qa/workunits/ceph-helpers.sh:988: wait_for_clean:  ((  timer >= 120  ))
       .../qa/workunits/ceph-helpers.sh:993: wait_for_clean:  get_num_active_clean
       .../qa/workunits/ceph-helpers.sh:863: get_num_active_clean:  local 'expression=('
       .../qa/workunits/ceph-helpers.sh:864: get_num_active_clean:  expression+='contains(.,'\''active'\'') and '
       .../qa/workunits/ceph-helpers.sh:865: get_num_active_clean:  expression+='contains(.,'\''clean'\'') and '
       .../qa/workunits/ceph-helpers.sh:866: get_num_active_clean:  expression+='not(contains(.,'\''stale'\''))'
       .../qa/workunits/ceph-helpers.sh:867: get_num_active_clean:  expression+=')'
       .../qa/workunits/ceph-helpers.sh:871: get_num_active_clean:  ceph --format xml pg dump pgs
       .../qa/workunits/ceph-helpers.sh:872: get_num_active_clean:  xmlstarlet sel -t -m '//pg_stat/state[(contains(.,'\''active'\'') and contains(.,'\''clean'\'') and not(contains(.,'\''stale'\'')))]' -v . -n
       .../qa/workunits/ceph-helpers.sh:873: get_num_active_clean:  grep -v '^$'
       .../qa/workunits/ceph-helpers.sh:873: get_num_active_clean:  wc -l
       ../qa/workunits/ceph-helpers.sh:993: wait_for_clean:  cur_active_clean=0
       ../qa/workunits/ceph-helpers.sh:994: wait_for_clean:  test 0 '!=' 0
       ../qa/workunits/ceph-helpers.sh:998: wait_for_clean:  sleep 1
       .../qa/workunits/ceph-helpers.sh:999: wait_for_clean:  expr 44 + 1
       ../qa/workunits/ceph-helpers.sh:999: wait_for_clean:  timer=45
       ../qa/workunits/ceph-helpers.sh:985: wait_for_clean:  is_clean
       .../qa/workunits/ceph-helpers.sh:952: is_clean:  get_num_pgs
       .../qa/workunits/ceph-helpers.sh:898: get_num_pgs:  ceph --format xml status
       .../qa/workunits/ceph-helpers.sh:899: get_num_pgs:  xmlstarlet sel -t -m //pgmap/num_pgs -v .
       ../qa/workunits/ceph-helpers.sh:952: is_clean:  num_pgs=4
       ../qa/workunits/ceph-helpers.sh:953: is_clean:  test 4 '!=' 0
       .../qa/workunits/ceph-helpers.sh:954: is_clean:  get_num_active_clean
       .../qa/workunits/ceph-helpers.sh:863: get_num_active_clean:  local 'expression=('
       .../qa/workunits/ceph-helpers.sh:864: get_num_active_clean:  expression+='contains(.,'\''active'\'') and '
       .../qa/workunits/ceph-helpers.sh:865: get_num_active_clean:  expression+='contains(.,'\''clean'\'') and '
       .../qa/workunits/ceph-helpers.sh:866: get_num_active_clean:  expression+='not(contains(.,'\''stale'\''))'
       .../qa/workunits/ceph-helpers.sh:867: get_num_active_clean:  expression+=')'
       .../qa/workunits/ceph-helpers.sh:871: get_num_active_clean:  ceph --format xml pg dump pgs
       .../qa/workunits/ceph-helpers.sh:872: get_num_active_clean:  xmlstarlet sel -t -m '//pg_stat/state[(contains(.,'\''active'\'') and contains(.,'\''clean'\'') and not(contains(.,'\''stale'\'')))]' -v . -n
       .../qa/workunits/ceph-helpers.sh:873: get_num_active_clean:  grep -v '^$'
       .../qa/workunits/ceph-helpers.sh:873: get_num_active_clean:  wc -l
       ../qa/workunits/ceph-helpers.sh:954: is_clean:  test 0 = 4
       ../qa/workunits/ceph-helpers.sh:954: is_clean:  return 1
       ../qa/workunits/ceph-helpers.sh:986: wait_for_clean:  get_is_making_recovery_progress
       .../qa/workunits/ceph-helpers.sh:839: get_is_making_recovery_progress:  xmlstarlet sel -t -m //pgmap/recovering_keys_per_sec -v . -o ' ' -t -m //pgmap/recovering_bytes_per_sec -v . -o ' ' -t -m //pgmap/recovering_objects_per_sec -v .
       .../qa/workunits/ceph-helpers.sh:839: get_is_making_recovery_progress:  ceph --format xml status
       ../qa/workunits/ceph-helpers.sh:839: get_is_making_recovery_progress:  local progress=
       ../qa/workunits/ceph-helpers.sh:840: get_is_making_recovery_progress:  test -n ''
       ../qa/workunits/ceph-helpers.sh:988: wait_for_clean:  ((  timer >= 120  ))
       .../qa/workunits/ceph-helpers.sh:993: wait_for_clean:  get_num_active_clean
       .../qa/workunits/ceph-helpers.sh:863: get_num_active_clean:  local 'expression=('
       .../qa/workunits/ceph-helpers.sh:864: get_num_active_clean:  expression+='contains(.,'\''active'\'') and '
       .../qa/workunits/ceph-helpers.sh:865: get_num_active_clean:  expression+='contains(.,'\''clean'\'') and '
       .../qa/workunits/ceph-helpers.sh:866: get_num_active_clean:  expression+='not(contains(.,'\''stale'\''))'
       .../qa/workunits/ceph-helpers.sh:867: get_num_active_clean:  expression+=')'
       .../qa/workunits/ceph-helpers.sh:871: get_num_active_clean:  ceph --format xml pg dump pgs
       .../qa/workunits/ceph-helpers.sh:872: get_num_active_clean:  xmlstarlet sel -t -m '//pg_stat/state[(contains(.,'\''active'\'') and contains(.,'\''clean'\'') and not(contains(.,'\''stale'\'')))]' -v . -n
       .../qa/workunits/ceph-helpers.sh:873: get_num_active_clean:  grep -v '^$'
       .../qa/workunits/ceph-helpers.sh:873: get_num_active_clean:  wc -l
       ../qa/workunits/ceph-helpers.sh:993: wait_for_clean:  cur_active_clean=0
       ../qa/workunits/ceph-helpers.sh:994: wait_for_clean:  test 0 '!=' 0
       ../qa/workunits/ceph-helpers.sh:998: wait_for_clean:  sleep 1
       .../qa/workunits/ceph-helpers.sh:999: wait_for_clean:  expr 45 + 1
       ../qa/workunits/ceph-helpers.sh:999: wait_for_clean:  timer=46
       ../qa/workunits/ceph-helpers.sh:985: wait_for_clean:  is_clean
       .../qa/workunits/ceph-helpers.sh:952: is_clean:  get_num_pgs
       .../qa/workunits/ceph-helpers.sh:898: get_num_pgs:  ceph --format xml status
       .../qa/workunits/ceph-helpers.sh:899: get_num_pgs:  xmlstarlet sel -t -m //pgmap/num_pgs -v .
       ../qa/workunits/ceph-helpers.sh:952: is_clean:  num_pgs=4
       ../qa/workunits/ceph-helpers.sh:953: is_clean:  test 4 '!=' 0
       .../qa/workunits/ceph-helpers.sh:954: is_clean:  get_num_active_clean
       .../qa/workunits/ceph-helpers.sh:863: get_num_active_clean:  local 'expression=('
       .../qa/workunits/ceph-helpers.sh:864: get_num_active_clean:  expression+='contains(.,'\''active'\'') and '
       .../qa/workunits/ceph-helpers.sh:865: get_num_active_clean:  expression+='contains(.,'\''clean'\'') and '
       .../qa/workunits/ceph-helpers.sh:866: get_num_active_clean:  expression+='not(contains(.,'\''stale'\''))'
       .../qa/workunits/ceph-helpers.sh:867: get_num_active_clean:  expression+=')'
       .../qa/workunits/ceph-helpers.sh:871: get_num_active_clean:  ceph --format xml pg dump pgs
       .../qa/workunits/ceph-helpers.sh:872: get_num_active_clean:  xmlstarlet sel -t -m '//pg_stat/state[(contains(.,'\''active'\'') and contains(.,'\''clean'\'') and not(contains(.,'\''stale'\'')))]' -v . -n
       .../qa/workunits/ceph-helpers.sh:873: get_num_active_clean:  grep -v '^$'
       .../qa/workunits/ceph-helpers.sh:873: get_num_active_clean:  wc -l
       ../qa/workunits/ceph-helpers.sh:954: is_clean:  test 0 = 4
       ../qa/workunits/ceph-helpers.sh:954: is_clean:  return 1
       ../qa/workunits/ceph-helpers.sh:986: wait_for_clean:  get_is_making_recovery_progress
       .../qa/workunits/ceph-helpers.sh:839: get_is_making_recovery_progress:  ceph --format xml status
       .../qa/workunits/ceph-helpers.sh:839: get_is_making_recovery_progress:  xmlstarlet sel -t -m //pgmap/recovering_keys_per_sec -v . -o ' ' -t -m //pgmap/recovering_bytes_per_sec -v . -o ' ' -t -m //pgmap/recovering_objects_per_sec -v .
       ../qa/workunits/ceph-helpers.sh:839: get_is_making_recovery_progress:  local progress=
       ../qa/workunits/ceph-helpers.sh:840: get_is_making_recovery_progress:  test -n ''
       ../qa/workunits/ceph-helpers.sh:988: wait_for_clean:  ((  timer >= 120  ))
       .../qa/workunits/ceph-helpers.sh:993: wait_for_clean:  get_num_active_clean
       .../qa/workunits/ceph-helpers.sh:863: get_num_active_clean:  local 'expression=('
       .../qa/workunits/ceph-helpers.sh:864: get_num_active_clean:  expression+='contains(.,'\''active'\'') and '
       .../qa/workunits/ceph-helpers.sh:865: get_num_active_clean:  expression+='contains(.,'\''clean'\'') and '
       .../qa/workunits/ceph-helpers.sh:866: get_num_active_clean:  expression+='not(contains(.,'\''stale'\''))'
       .../qa/workunits/ceph-helpers.sh:867: get_num_active_clean:  expression+=')'
       .../qa/workunits/ceph-helpers.sh:871: get_num_active_clean:  ceph --format xml pg dump pgs
       .../qa/workunits/ceph-helpers.sh:872: get_num_active_clean:  xmlstarlet sel -t -m '//pg_stat/state[(contains(.,'\''active'\'') and contains(.,'\''clean'\'') and not(contains(.,'\''stale'\'')))]' -v . -n
       .../qa/workunits/ceph-helpers.sh:873: get_num_active_clean:  grep -v '^$'
       .../qa/workunits/ceph-helpers.sh:873: get_num_active_clean:  wc -l
       ../qa/workunits/ceph-helpers.sh:993: wait_for_clean:  cur_active_clean=0
       ../qa/workunits/ceph-helpers.sh:994: wait_for_clean:  test 0 '!=' 0
       ../qa/workunits/ceph-helpers.sh:998: wait_for_clean:  sleep 1
       .../qa/workunits/ceph-helpers.sh:999: wait_for_clean:  expr 46 + 1
       ../qa/workunits/ceph-helpers.sh:999: wait_for_clean:  timer=47
       ../qa/workunits/ceph-helpers.sh:985: wait_for_clean:  is_clean
       .../qa/workunits/ceph-helpers.sh:952: is_clean:  get_num_pgs
       .../qa/workunits/ceph-helpers.sh:898: get_num_pgs:  ceph --format xml status
       .../qa/workunits/ceph-helpers.sh:899: get_num_pgs:  xmlstarlet sel -t -m //pgmap/num_pgs -v .
       ../qa/workunits/ceph-helpers.sh:952: is_clean:  num_pgs=4
       ../qa/workunits/ceph-helpers.sh:953: is_clean:  test 4 '!=' 0
       .../qa/workunits/ceph-helpers.sh:954: is_clean:  get_num_active_clean
       .../qa/workunits/ceph-helpers.sh:863: get_num_active_clean:  local 'expression=('
       .../qa/workunits/ceph-helpers.sh:864: get_num_active_clean:  expression+='contains(.,'\''active'\'') and '
       .../qa/workunits/ceph-helpers.sh:865: get_num_active_clean:  expression+='contains(.,'\''clean'\'') and '
       .../qa/workunits/ceph-helpers.sh:866: get_num_active_clean:  expression+='not(contains(.,'\''stale'\''))'
       .../qa/workunits/ceph-helpers.sh:867: get_num_active_clean:  expression+=')'
       .../qa/workunits/ceph-helpers.sh:871: get_num_active_clean:  ceph --format xml pg dump pgs
       .../qa/workunits/ceph-helpers.sh:872: get_num_active_clean:  xmlstarlet sel -t -m '//pg_stat/state[(contains(.,'\''active'\'') and contains(.,'\''clean'\'') and not(contains(.,'\''stale'\'')))]' -v . -n
       .../qa/workunits/ceph-helpers.sh:873: get_num_active_clean:  grep -v '^$'
       .../qa/workunits/ceph-helpers.sh:873: get_num_active_clean:  wc -l
       ../qa/workunits/ceph-helpers.sh:954: is_clean:  test 0 = 4
       ../qa/workunits/ceph-helpers.sh:954: is_clean:  return 1
       ../qa/workunits/ceph-helpers.sh:986: wait_for_clean:  get_is_making_recovery_progress
       .../qa/workunits/ceph-helpers.sh:839: get_is_making_recovery_progress:  ceph --format xml status
       .../qa/workunits/ceph-helpers.sh:839: get_is_making_recovery_progress:  xmlstarlet sel -t -m //pgmap/recovering_keys_per_sec -v . -o ' ' -t -m //pgmap/recovering_bytes_per_sec -v . -o ' ' -t -m //pgmap/recovering_objects_per_sec -v .
       ../qa/workunits/ceph-helpers.sh:839: get_is_making_recovery_progress:  local progress=
       ../qa/workunits/ceph-helpers.sh:840: get_is_making_recovery_progress:  test -n ''
       ../qa/workunits/ceph-helpers.sh:988: wait_for_clean:  ((  timer >= 120  ))
       .../qa/workunits/ceph-helpers.sh:993: wait_for_clean:  get_num_active_clean
       .../qa/workunits/ceph-helpers.sh:863: get_num_active_clean:  local 'expression=('
       .../qa/workunits/ceph-helpers.sh:864: get_num_active_clean:  expression+='contains(.,'\''active'\'') and '
       .../qa/workunits/ceph-helpers.sh:865: get_num_active_clean:  expression+='contains(.,'\''clean'\'') and '
       .../qa/workunits/ceph-helpers.sh:866: get_num_active_clean:  expression+='not(contains(.,'\''stale'\''))'
       .../qa/workunits/ceph-helpers.sh:867: get_num_active_clean:  expression+=')'
       .../qa/workunits/ceph-helpers.sh:871: get_num_active_clean:  ceph --format xml pg dump pgs
       .../qa/workunits/ceph-helpers.sh:872: get_num_active_clean:  xmlstarlet sel -t -m '//pg_stat/state[(contains(.,'\''active'\'') and contains(.,'\''clean'\'') and not(contains(.,'\''stale'\'')))]' -v . -n
       .../qa/workunits/ceph-helpers.sh:873: get_num_active_clean:  grep -v '^$'
       .../qa/workunits/ceph-helpers.sh:873: get_num_active_clean:  wc -l
       ../qa/workunits/ceph-helpers.sh:993: wait_for_clean:  cur_active_clean=0
       ../qa/workunits/ceph-helpers.sh:994: wait_for_clean:  test 0 '!=' 0
       ../qa/workunits/ceph-helpers.sh:998: wait_for_clean:  sleep 1
       .../qa/workunits/ceph-helpers.sh:999: wait_for_clean:  expr 47 + 1
       ../qa/workunits/ceph-helpers.sh:999: wait_for_clean:  timer=48
       ../qa/workunits/ceph-helpers.sh:985: wait_for_clean:  is_clean
       .../qa/workunits/ceph-helpers.sh:952: is_clean:  get_num_pgs
       .../qa/workunits/ceph-helpers.sh:898: get_num_pgs:  ceph --format xml status
       .../qa/workunits/ceph-helpers.sh:899: get_num_pgs:  xmlstarlet sel -t -m //pgmap/num_pgs -v .
       ../qa/workunits/ceph-helpers.sh:952: is_clean:  num_pgs=4
       ../qa/workunits/ceph-helpers.sh:953: is_clean:  test 4 '!=' 0
       .../qa/workunits/ceph-helpers.sh:954: is_clean:  get_num_active_clean
       .../qa/workunits/ceph-helpers.sh:863: get_num_active_clean:  local 'expression=('
       .../qa/workunits/ceph-helpers.sh:864: get_num_active_clean:  expression+='contains(.,'\''active'\'') and '
       .../qa/workunits/ceph-helpers.sh:865: get_num_active_clean:  expression+='contains(.,'\''clean'\'') and '
       .../qa/workunits/ceph-helpers.sh:866: get_num_active_clean:  expression+='not(contains(.,'\''stale'\''))'
       .../qa/workunits/ceph-helpers.sh:867: get_num_active_clean:  expression+=')'
       .../qa/workunits/ceph-helpers.sh:871: get_num_active_clean:  ceph --format xml pg dump pgs
       .../qa/workunits/ceph-helpers.sh:872: get_num_active_clean:  xmlstarlet sel -t -m '//pg_stat/state[(contains(.,'\''active'\'') and contains(.,'\''clean'\'') and not(contains(.,'\''stale'\'')))]' -v . -n
       .../qa/workunits/ceph-helpers.sh:873: get_num_active_clean:  grep -v '^$'
       .../qa/workunits/ceph-helpers.sh:873: get_num_active_clean:  wc -l
       ../qa/workunits/ceph-helpers.sh:954: is_clean:  test 0 = 4
       ../qa/workunits/ceph-helpers.sh:954: is_clean:  return 1
       ../qa/workunits/ceph-helpers.sh:986: wait_for_clean:  get_is_making_recovery_progress
       .../qa/workunits/ceph-helpers.sh:839: get_is_making_recovery_progress:  ceph --format xml status
       .../qa/workunits/ceph-helpers.sh:839: get_is_making_recovery_progress:  xmlstarlet sel -t -m //pgmap/recovering_keys_per_sec -v . -o ' ' -t -m //pgmap/recovering_bytes_per_sec -v . -o ' ' -t -m //pgmap/recovering_objects_per_sec -v .
       ../qa/workunits/ceph-helpers.sh:839: get_is_making_recovery_progress:  local progress=
       ../qa/workunits/ceph-helpers.sh:840: get_is_making_recovery_progress:  test -n ''
       ../qa/workunits/ceph-helpers.sh:988: wait_for_clean:  ((  timer >= 120  ))
       .../qa/workunits/ceph-helpers.sh:993: wait_for_clean:  get_num_active_clean
       .../qa/workunits/ceph-helpers.sh:863: get_num_active_clean:  local 'expression=('
       .../qa/workunits/ceph-helpers.sh:864: get_num_active_clean:  expression+='contains(.,'\''active'\'') and '
       .../qa/workunits/ceph-helpers.sh:865: get_num_active_clean:  expression+='contains(.,'\''clean'\'') and '
       .../qa/workunits/ceph-helpers.sh:866: get_num_active_clean:  expression+='not(contains(.,'\''stale'\''))'
       .../qa/workunits/ceph-helpers.sh:867: get_num_active_clean:  expression+=')'
       .../qa/workunits/ceph-helpers.sh:871: get_num_active_clean:  ceph --format xml pg dump pgs
       .../qa/workunits/ceph-helpers.sh:872: get_num_active_clean:  xmlstarlet sel -t -m '//pg_stat/state[(contains(.,'\''active'\'') and contains(.,'\''clean'\'') and not(contains(.,'\''stale'\'')))]' -v . -n
       .../qa/workunits/ceph-helpers.sh:873: get_num_active_clean:  grep -v '^$'
       .../qa/workunits/ceph-helpers.sh:873: get_num_active_clean:  wc -l
       ../qa/workunits/ceph-helpers.sh:993: wait_for_clean:  cur_active_clean=0
       ../qa/workunits/ceph-helpers.sh:994: wait_for_clean:  test 0 '!=' 0
       ../qa/workunits/ceph-helpers.sh:998: wait_for_clean:  sleep 1
       .../qa/workunits/ceph-helpers.sh:999: wait_for_clean:  expr 48 + 1
       ../qa/workunits/ceph-helpers.sh:999: wait_for_clean:  timer=49
       ../qa/workunits/ceph-helpers.sh:985: wait_for_clean:  is_clean
       .../qa/workunits/ceph-helpers.sh:952: is_clean:  get_num_pgs
       .../qa/workunits/ceph-helpers.sh:898: get_num_pgs:  ceph --format xml status
       .../qa/workunits/ceph-helpers.sh:899: get_num_pgs:  xmlstarlet sel -t -m //pgmap/num_pgs -v .
       ../qa/workunits/ceph-helpers.sh:952: is_clean:  num_pgs=4
       ../qa/workunits/ceph-helpers.sh:953: is_clean:  test 4 '!=' 0
       .../qa/workunits/ceph-helpers.sh:954: is_clean:  get_num_active_clean
       .../qa/workunits/ceph-helpers.sh:863: get_num_active_clean:  local 'expression=('
       .../qa/workunits/ceph-helpers.sh:864: get_num_active_clean:  expression+='contains(.,'\''active'\'') and '
       .../qa/workunits/ceph-helpers.sh:865: get_num_active_clean:  expression+='contains(.,'\''clean'\'') and '
       .../qa/workunits/ceph-helpers.sh:866: get_num_active_clean:  expression+='not(contains(.,'\''stale'\''))'
       .../qa/workunits/ceph-helpers.sh:867: get_num_active_clean:  expression+=')'
       .../qa/workunits/ceph-helpers.sh:871: get_num_active_clean:  ceph --format xml pg dump pgs
       .../qa/workunits/ceph-helpers.sh:872: get_num_active_clean:  xmlstarlet sel -t -m '//pg_stat/state[(contains(.,'\''active'\'') and contains(.,'\''clean'\'') and not(contains(.,'\''stale'\'')))]' -v . -n
       .../qa/workunits/ceph-helpers.sh:873: get_num_active_clean:  wc -l
       .../qa/workunits/ceph-helpers.sh:873: get_num_active_clean:  grep -v '^$'
       ../qa/workunits/ceph-helpers.sh:954: is_clean:  test 0 = 4
       ../qa/workunits/ceph-helpers.sh:954: is_clean:  return 1
       ../qa/workunits/ceph-helpers.sh:986: wait_for_clean:  get_is_making_recovery_progress
       .../qa/workunits/ceph-helpers.sh:839: get_is_making_recovery_progress:  ceph --format xml status
       .../qa/workunits/ceph-helpers.sh:839: get_is_making_recovery_progress:  xmlstarlet sel -t -m //pgmap/recovering_keys_per_sec -v . -o ' ' -t -m //pgmap/recovering_bytes_per_sec -v . -o ' ' -t -m //pgmap/recovering_objects_per_sec -v .
       ../qa/workunits/ceph-helpers.sh:839: get_is_making_recovery_progress:  local progress=
       ../qa/workunits/ceph-helpers.sh:840: get_is_making_recovery_progress:  test -n ''
       ../qa/workunits/ceph-helpers.sh:988: wait_for_clean:  ((  timer >= 120  ))
       .../qa/workunits/ceph-helpers.sh:993: wait_for_clean:  get_num_active_clean
       .../qa/workunits/ceph-helpers.sh:863: get_num_active_clean:  local 'expression=('
       .../qa/workunits/ceph-helpers.sh:864: get_num_active_clean:  expression+='contains(.,'\''active'\'') and '
       .../qa/workunits/ceph-helpers.sh:865: get_num_active_clean:  expression+='contains(.,'\''clean'\'') and '
       .../qa/workunits/ceph-helpers.sh:866: get_num_active_clean:  expression+='not(contains(.,'\''stale'\''))'
       .../qa/workunits/ceph-helpers.sh:867: get_num_active_clean:  expression+=')'
       .../qa/workunits/ceph-helpers.sh:871: get_num_active_clean:  ceph --format xml pg dump pgs
       .../qa/workunits/ceph-helpers.sh:872: get_num_active_clean:  xmlstarlet sel -t -m '//pg_stat/state[(contains(.,'\''active'\'') and contains(.,'\''clean'\'') and not(contains(.,'\''stale'\'')))]' -v . -n
       .../qa/workunits/ceph-helpers.sh:873: get_num_active_clean:  grep -v '^$'
       .../qa/workunits/ceph-helpers.sh:873: get_num_active_clean:  wc -l
       ../qa/workunits/ceph-helpers.sh:993: wait_for_clean:  cur_active_clean=0
       ../qa/workunits/ceph-helpers.sh:994: wait_for_clean:  test 0 '!=' 0
       ../qa/workunits/ceph-helpers.sh:998: wait_for_clean:  sleep 1
       .../qa/workunits/ceph-helpers.sh:999: wait_for_clean:  expr 49 + 1
       ../qa/workunits/ceph-helpers.sh:999: wait_for_clean:  timer=50
       ../qa/workunits/ceph-helpers.sh:985: wait_for_clean:  is_clean
       .../qa/workunits/ceph-helpers.sh:952: is_clean:  get_num_pgs
       .../qa/workunits/ceph-helpers.sh:898: get_num_pgs:  ceph --format xml status
       .../qa/workunits/ceph-helpers.sh:899: get_num_pgs:  xmlstarlet sel -t -m //pgmap/num_pgs -v .
       ../qa/workunits/ceph-helpers.sh:952: is_clean:  num_pgs=4
       ../qa/workunits/ceph-helpers.sh:953: is_clean:  test 4 '!=' 0
       .../qa/workunits/ceph-helpers.sh:954: is_clean:  get_num_active_clean
       .../qa/workunits/ceph-helpers.sh:863: get_num_active_clean:  local 'expression=('
       .../qa/workunits/ceph-helpers.sh:864: get_num_active_clean:  expression+='contains(.,'\''active'\'') and '
       .../qa/workunits/ceph-helpers.sh:865: get_num_active_clean:  expression+='contains(.,'\''clean'\'') and '
       .../qa/workunits/ceph-helpers.sh:866: get_num_active_clean:  expression+='not(contains(.,'\''stale'\''))'
       .../qa/workunits/ceph-helpers.sh:867: get_num_active_clean:  expression+=')'
       .../qa/workunits/ceph-helpers.sh:871: get_num_active_clean:  ceph --format xml pg dump pgs
       .../qa/workunits/ceph-helpers.sh:872: get_num_active_clean:  xmlstarlet sel -t -m '//pg_stat/state[(contains(.,'\''active'\'') and contains(.,'\''clean'\'') and not(contains(.,'\''stale'\'')))]' -v . -n
       .../qa/workunits/ceph-helpers.sh:873: get_num_active_clean:  grep -v '^$'
       .../qa/workunits/ceph-helpers.sh:873: get_num_active_clean:  wc -l
       ../qa/workunits/ceph-helpers.sh:954: is_clean:  test 0 = 4
       ../qa/workunits/ceph-helpers.sh:954: is_clean:  return 1
       ../qa/workunits/ceph-helpers.sh:986: wait_for_clean:  get_is_making_recovery_progress
       .../qa/workunits/ceph-helpers.sh:839: get_is_making_recovery_progress:  ceph --format xml status
       .../qa/workunits/ceph-helpers.sh:839: get_is_making_recovery_progress:  xmlstarlet sel -t -m //pgmap/recovering_keys_per_sec -v . -o ' ' -t -m //pgmap/recovering_bytes_per_sec -v . -o ' ' -t -m //pgmap/recovering_objects_per_sec -v .
       ../qa/workunits/ceph-helpers.sh:839: get_is_making_recovery_progress:  local progress=
       ../qa/workunits/ceph-helpers.sh:840: get_is_making_recovery_progress:  test -n ''
       ../qa/workunits/ceph-helpers.sh:988: wait_for_clean:  ((  timer >= 120  ))
       .../qa/workunits/ceph-helpers.sh:993: wait_for_clean:  get_num_active_clean
       .../qa/workunits/ceph-helpers.sh:863: get_num_active_clean:  local 'expression=('
       .../qa/workunits/ceph-helpers.sh:864: get_num_active_clean:  expression+='contains(.,'\''active'\'') and '
       .../qa/workunits/ceph-helpers.sh:865: get_num_active_clean:  expression+='contains(.,'\''clean'\'') and '
       .../qa/workunits/ceph-helpers.sh:866: get_num_active_clean:  expression+='not(contains(.,'\''stale'\''))'
       .../qa/workunits/ceph-helpers.sh:867: get_num_active_clean:  expression+=')'
       .../qa/workunits/ceph-helpers.sh:871: get_num_active_clean:  ceph --format xml pg dump pgs
       .../qa/workunits/ceph-helpers.sh:872: get_num_active_clean:  xmlstarlet sel -t -m '//pg_stat/state[(contains(.,'\''active'\'') and contains(.,'\''clean'\'') and not(contains(.,'\''stale'\'')))]' -v . -n
       .../qa/workunits/ceph-helpers.sh:873: get_num_active_clean:  grep -v '^$'
       .../qa/workunits/ceph-helpers.sh:873: get_num_active_clean:  wc -l
       ../qa/workunits/ceph-helpers.sh:993: wait_for_clean:  cur_active_clean=0
       ../qa/workunits/ceph-helpers.sh:994: wait_for_clean:  test 0 '!=' 0
       ../qa/workunits/ceph-helpers.sh:998: wait_for_clean:  sleep 1
       .../qa/workunits/ceph-helpers.sh:999: wait_for_clean:  expr 50 + 1
       ../qa/workunits/ceph-helpers.sh:999: wait_for_clean:  timer=51
       ../qa/workunits/ceph-helpers.sh:985: wait_for_clean:  is_clean
       .../qa/workunits/ceph-helpers.sh:952: is_clean:  get_num_pgs
       .../qa/workunits/ceph-helpers.sh:898: get_num_pgs:  ceph --format xml status
       .../qa/workunits/ceph-helpers.sh:899: get_num_pgs:  xmlstarlet sel -t -m //pgmap/num_pgs -v .
       ../qa/workunits/ceph-helpers.sh:952: is_clean:  num_pgs=4
       ../qa/workunits/ceph-helpers.sh:953: is_clean:  test 4 '!=' 0
       .../qa/workunits/ceph-helpers.sh:954: is_clean:  get_num_active_clean
       .../qa/workunits/ceph-helpers.sh:863: get_num_active_clean:  local 'expression=('
       .../qa/workunits/ceph-helpers.sh:864: get_num_active_clean:  expression+='contains(.,'\''active'\'') and '
       .../qa/workunits/ceph-helpers.sh:865: get_num_active_clean:  expression+='contains(.,'\''clean'\'') and '
       .../qa/workunits/ceph-helpers.sh:866: get_num_active_clean:  expression+='not(contains(.,'\''stale'\''))'
       .../qa/workunits/ceph-helpers.sh:867: get_num_active_clean:  expression+=')'
       .../qa/workunits/ceph-helpers.sh:871: get_num_active_clean:  ceph --format xml pg dump pgs
       .../qa/workunits/ceph-helpers.sh:872: get_num_active_clean:  xmlstarlet sel -t -m '//pg_stat/state[(contains(.,'\''active'\'') and contains(.,'\''clean'\'') and not(contains(.,'\''stale'\'')))]' -v . -n
       .../qa/workunits/ceph-helpers.sh:873: get_num_active_clean:  grep -v '^$'
       .../qa/workunits/ceph-helpers.sh:873: get_num_active_clean:  wc -l
       ../qa/workunits/ceph-helpers.sh:954: is_clean:  test 0 = 4
       ../qa/workunits/ceph-helpers.sh:954: is_clean:  return 1
       ../qa/workunits/ceph-helpers.sh:986: wait_for_clean:  get_is_making_recovery_progress
       .../qa/workunits/ceph-helpers.sh:839: get_is_making_recovery_progress:  ceph --format xml status
       .../qa/workunits/ceph-helpers.sh:839: get_is_making_recovery_progress:  xmlstarlet sel -t -m //pgmap/recovering_keys_per_sec -v . -o ' ' -t -m //pgmap/recovering_bytes_per_sec -v . -o ' ' -t -m //pgmap/recovering_objects_per_sec -v .
       ../qa/workunits/ceph-helpers.sh:839: get_is_making_recovery_progress:  local progress=
       ../qa/workunits/ceph-helpers.sh:840: get_is_making_recovery_progress:  test -n ''
       ../qa/workunits/ceph-helpers.sh:988: wait_for_clean:  ((  timer >= 120  ))
       .../qa/workunits/ceph-helpers.sh:993: wait_for_clean:  get_num_active_clean
       .../qa/workunits/ceph-helpers.sh:863: get_num_active_clean:  local 'expression=('
       .../qa/workunits/ceph-helpers.sh:864: get_num_active_clean:  expression+='contains(.,'\''active'\'') and '
       .../qa/workunits/ceph-helpers.sh:865: get_num_active_clean:  expression+='contains(.,'\''clean'\'') and '
       .../qa/workunits/ceph-helpers.sh:866: get_num_active_clean:  expression+='not(contains(.,'\''stale'\''))'
       .../qa/workunits/ceph-helpers.sh:867: get_num_active_clean:  expression+=')'
       .../qa/workunits/ceph-helpers.sh:871: get_num_active_clean:  ceph --format xml pg dump pgs
       .../qa/workunits/ceph-helpers.sh:872: get_num_active_clean:  xmlstarlet sel -t -m '//pg_stat/state[(contains(.,'\''active'\'') and contains(.,'\''clean'\'') and not(contains(.,'\''stale'\'')))]' -v . -n
       .../qa/workunits/ceph-helpers.sh:873: get_num_active_clean:  grep -v '^$'
       .../qa/workunits/ceph-helpers.sh:873: get_num_active_clean:  wc -l
       ../qa/workunits/ceph-helpers.sh:993: wait_for_clean:  cur_active_clean=0
       ../qa/workunits/ceph-helpers.sh:994: wait_for_clean:  test 0 '!=' 0
       ../qa/workunits/ceph-helpers.sh:998: wait_for_clean:  sleep 1
       .../qa/workunits/ceph-helpers.sh:999: wait_for_clean:  expr 51 + 1
       ../qa/workunits/ceph-helpers.sh:999: wait_for_clean:  timer=52
       ../qa/workunits/ceph-helpers.sh:985: wait_for_clean:  is_clean
       .../qa/workunits/ceph-helpers.sh:952: is_clean:  get_num_pgs
       .../qa/workunits/ceph-helpers.sh:898: get_num_pgs:  ceph --format xml status
       .../qa/workunits/ceph-helpers.sh:899: get_num_pgs:  xmlstarlet sel -t -m //pgmap/num_pgs -v .
       ../qa/workunits/ceph-helpers.sh:952: is_clean:  num_pgs=4
       ../qa/workunits/ceph-helpers.sh:953: is_clean:  test 4 '!=' 0
       .../qa/workunits/ceph-helpers.sh:954: is_clean:  get_num_active_clean
       .../qa/workunits/ceph-helpers.sh:863: get_num_active_clean:  local 'expression=('
       .../qa/workunits/ceph-helpers.sh:864: get_num_active_clean:  expression+='contains(.,'\''active'\'') and '
       .../qa/workunits/ceph-helpers.sh:865: get_num_active_clean:  expression+='contains(.,'\''clean'\'') and '
       .../qa/workunits/ceph-helpers.sh:866: get_num_active_clean:  expression+='not(contains(.,'\''stale'\''))'
       .../qa/workunits/ceph-helpers.sh:867: get_num_active_clean:  expression+=')'
       .../qa/workunits/ceph-helpers.sh:871: get_num_active_clean:  ceph --format xml pg dump pgs
       .../qa/workunits/ceph-helpers.sh:872: get_num_active_clean:  xmlstarlet sel -t -m '//pg_stat/state[(contains(.,'\''active'\'') and contains(.,'\''clean'\'') and not(contains(.,'\''stale'\'')))]' -v . -n
       .../qa/workunits/ceph-helpers.sh:873: get_num_active_clean:  wc -l
       .../qa/workunits/ceph-helpers.sh:873: get_num_active_clean:  grep -v '^$'
       ../qa/workunits/ceph-helpers.sh:954: is_clean:  test 0 = 4
       ../qa/workunits/ceph-helpers.sh:954: is_clean:  return 1
       ../qa/workunits/ceph-helpers.sh:986: wait_for_clean:  get_is_making_recovery_progress
       .../qa/workunits/ceph-helpers.sh:839: get_is_making_recovery_progress:  ceph --format xml status
       .../qa/workunits/ceph-helpers.sh:839: get_is_making_recovery_progress:  xmlstarlet sel -t -m //pgmap/recovering_keys_per_sec -v . -o ' ' -t -m //pgmap/recovering_bytes_per_sec -v . -o ' ' -t -m //pgmap/recovering_objects_per_sec -v .
       ../qa/workunits/ceph-helpers.sh:839: get_is_making_recovery_progress:  local progress=
       ../qa/workunits/ceph-helpers.sh:840: get_is_making_recovery_progress:  test -n ''
       ../qa/workunits/ceph-helpers.sh:988: wait_for_clean:  ((  timer >= 120  ))
       .../qa/workunits/ceph-helpers.sh:993: wait_for_clean:  get_num_active_clean
       .../qa/workunits/ceph-helpers.sh:863: get_num_active_clean:  local 'expression=('
       .../qa/workunits/ceph-helpers.sh:864: get_num_active_clean:  expression+='contains(.,'\''active'\'') and '
       .../qa/workunits/ceph-helpers.sh:865: get_num_active_clean:  expression+='contains(.,'\''clean'\'') and '
       .../qa/workunits/ceph-helpers.sh:866: get_num_active_clean:  expression+='not(contains(.,'\''stale'\''))'
       .../qa/workunits/ceph-helpers.sh:867: get_num_active_clean:  expression+=')'
       .../qa/workunits/ceph-helpers.sh:871: get_num_active_clean:  ceph --format xml pg dump pgs
       .../qa/workunits/ceph-helpers.sh:872: get_num_active_clean:  xmlstarlet sel -t -m '//pg_stat/state[(contains(.,'\''active'\'') and contains(.,'\''clean'\'') and not(contains(.,'\''stale'\'')))]' -v . -n
       .../qa/workunits/ceph-helpers.sh:873: get_num_active_clean:  grep -v '^$'
       .../qa/workunits/ceph-helpers.sh:873: get_num_active_clean:  wc -l
       ../qa/workunits/ceph-helpers.sh:993: wait_for_clean:  cur_active_clean=0
       ../qa/workunits/ceph-helpers.sh:994: wait_for_clean:  test 0 '!=' 0
       ../qa/workunits/ceph-helpers.sh:998: wait_for_clean:  sleep 1
       .../qa/workunits/ceph-helpers.sh:999: wait_for_clean:  expr 52 + 1
       ../qa/workunits/ceph-helpers.sh:999: wait_for_clean:  timer=53
       ../qa/workunits/ceph-helpers.sh:985: wait_for_clean:  is_clean
       .../qa/workunits/ceph-helpers.sh:952: is_clean:  get_num_pgs
       .../qa/workunits/ceph-helpers.sh:898: get_num_pgs:  ceph --format xml status
       .../qa/workunits/ceph-helpers.sh:899: get_num_pgs:  xmlstarlet sel -t -m //pgmap/num_pgs -v .
       ../qa/workunits/ceph-helpers.sh:952: is_clean:  num_pgs=4
       ../qa/workunits/ceph-helpers.sh:953: is_clean:  test 4 '!=' 0
       .../qa/workunits/ceph-helpers.sh:954: is_clean:  get_num_active_clean
       .../qa/workunits/ceph-helpers.sh:863: get_num_active_clean:  local 'expression=('
       .../qa/workunits/ceph-helpers.sh:864: get_num_active_clean:  expression+='contains(.,'\''active'\'') and '
       .../qa/workunits/ceph-helpers.sh:865: get_num_active_clean:  expression+='contains(.,'\''clean'\'') and '
       .../qa/workunits/ceph-helpers.sh:866: get_num_active_clean:  expression+='not(contains(.,'\''stale'\''))'
       .../qa/workunits/ceph-helpers.sh:867: get_num_active_clean:  expression+=')'
       .../qa/workunits/ceph-helpers.sh:871: get_num_active_clean:  ceph --format xml pg dump pgs
       .../qa/workunits/ceph-helpers.sh:872: get_num_active_clean:  xmlstarlet sel -t -m '//pg_stat/state[(contains(.,'\''active'\'') and contains(.,'\''clean'\'') and not(contains(.,'\''stale'\'')))]' -v . -n
       .../qa/workunits/ceph-helpers.sh:873: get_num_active_clean:  wc -l
       .../qa/workunits/ceph-helpers.sh:873: get_num_active_clean:  grep -v '^$'
       ../qa/workunits/ceph-helpers.sh:954: is_clean:  test 0 = 4
       ../qa/workunits/ceph-helpers.sh:954: is_clean:  return 1
       ../qa/workunits/ceph-helpers.sh:986: wait_for_clean:  get_is_making_recovery_progress
       .../qa/workunits/ceph-helpers.sh:839: get_is_making_recovery_progress:  ceph --format xml status
       .../qa/workunits/ceph-helpers.sh:839: get_is_making_recovery_progress:  xmlstarlet sel -t -m //pgmap/recovering_keys_per_sec -v . -o ' ' -t -m //pgmap/recovering_bytes_per_sec -v . -o ' ' -t -m //pgmap/recovering_objects_per_sec -v .
       ../qa/workunits/ceph-helpers.sh:839: get_is_making_recovery_progress:  local progress=
       ../qa/workunits/ceph-helpers.sh:840: get_is_making_recovery_progress:  test -n ''
       ../qa/workunits/ceph-helpers.sh:988: wait_for_clean:  ((  timer >= 120  ))
       .../qa/workunits/ceph-helpers.sh:993: wait_for_clean:  get_num_active_clean
       .../qa/workunits/ceph-helpers.sh:863: get_num_active_clean:  local 'expression=('
       .../qa/workunits/ceph-helpers.sh:864: get_num_active_clean:  expression+='contains(.,'\''active'\'') and '
       .../qa/workunits/ceph-helpers.sh:865: get_num_active_clean:  expression+='contains(.,'\''clean'\'') and '
       .../qa/workunits/ceph-helpers.sh:866: get_num_active_clean:  expression+='not(contains(.,'\''stale'\''))'
       .../qa/workunits/ceph-helpers.sh:867: get_num_active_clean:  expression+=')'
       .../qa/workunits/ceph-helpers.sh:871: get_num_active_clean:  ceph --format xml pg dump pgs
       .../qa/workunits/ceph-helpers.sh:872: get_num_active_clean:  xmlstarlet sel -t -m '//pg_stat/state[(contains(.,'\''active'\'') and contains(.,'\''clean'\'') and not(contains(.,'\''stale'\'')))]' -v . -n
       .../qa/workunits/ceph-helpers.sh:873: get_num_active_clean:  wc -l
       .../qa/workunits/ceph-helpers.sh:873: get_num_active_clean:  grep -v '^$'
       ../qa/workunits/ceph-helpers.sh:993: wait_for_clean:  cur_active_clean=0
       ../qa/workunits/ceph-helpers.sh:994: wait_for_clean:  test 0 '!=' 0
       ../qa/workunits/ceph-helpers.sh:998: wait_for_clean:  sleep 1
       .../qa/workunits/ceph-helpers.sh:999: wait_for_clean:  expr 53 + 1
       ../qa/workunits/ceph-helpers.sh:999: wait_for_clean:  timer=54
       ../qa/workunits/ceph-helpers.sh:985: wait_for_clean:  is_clean
       .../qa/workunits/ceph-helpers.sh:952: is_clean:  get_num_pgs
       .../qa/workunits/ceph-helpers.sh:898: get_num_pgs:  ceph --format xml status
       .../qa/workunits/ceph-helpers.sh:899: get_num_pgs:  xmlstarlet sel -t -m //pgmap/num_pgs -v .
       ../qa/workunits/ceph-helpers.sh:952: is_clean:  num_pgs=4
       ../qa/workunits/ceph-helpers.sh:953: is_clean:  test 4 '!=' 0
       .../qa/workunits/ceph-helpers.sh:954: is_clean:  get_num_active_clean
       .../qa/workunits/ceph-helpers.sh:863: get_num_active_clean:  local 'expression=('
       .../qa/workunits/ceph-helpers.sh:864: get_num_active_clean:  expression+='contains(.,'\''active'\'') and '
       .../qa/workunits/ceph-helpers.sh:865: get_num_active_clean:  expression+='contains(.,'\''clean'\'') and '
       .../qa/workunits/ceph-helpers.sh:866: get_num_active_clean:  expression+='not(contains(.,'\''stale'\''))'
       .../qa/workunits/ceph-helpers.sh:867: get_num_active_clean:  expression+=')'
       .../qa/workunits/ceph-helpers.sh:871: get_num_active_clean:  ceph --format xml pg dump pgs
       .../qa/workunits/ceph-helpers.sh:872: get_num_active_clean:  xmlstarlet sel -t -m '//pg_stat/state[(contains(.,'\''active'\'') and contains(.,'\''clean'\'') and not(contains(.,'\''stale'\'')))]' -v . -n
       .../qa/workunits/ceph-helpers.sh:873: get_num_active_clean:  grep -v '^$'
       .../qa/workunits/ceph-helpers.sh:873: get_num_active_clean:  wc -l
       ../qa/workunits/ceph-helpers.sh:954: is_clean:  test 0 = 4
       ../qa/workunits/ceph-helpers.sh:954: is_clean:  return 1
       ../qa/workunits/ceph-helpers.sh:986: wait_for_clean:  get_is_making_recovery_progress
       .../qa/workunits/ceph-helpers.sh:839: get_is_making_recovery_progress:  ceph --format xml status
       .../qa/workunits/ceph-helpers.sh:839: get_is_making_recovery_progress:  xmlstarlet sel -t -m //pgmap/recovering_keys_per_sec -v . -o ' ' -t -m //pgmap/recovering_bytes_per_sec -v . -o ' ' -t -m //pgmap/recovering_objects_per_sec -v .
       ../qa/workunits/ceph-helpers.sh:839: get_is_making_recovery_progress:  local progress=
       ../qa/workunits/ceph-helpers.sh:840: get_is_making_recovery_progress:  test -n ''
       ../qa/workunits/ceph-helpers.sh:988: wait_for_clean:  ((  timer >= 120  ))
       .../qa/workunits/ceph-helpers.sh:993: wait_for_clean:  get_num_active_clean
       .../qa/workunits/ceph-helpers.sh:863: get_num_active_clean:  local 'expression=('
       .../qa/workunits/ceph-helpers.sh:864: get_num_active_clean:  expression+='contains(.,'\''active'\'') and '
       .../qa/workunits/ceph-helpers.sh:865: get_num_active_clean:  expression+='contains(.,'\''clean'\'') and '
       .../qa/workunits/ceph-helpers.sh:866: get_num_active_clean:  expression+='not(contains(.,'\''stale'\''))'
       .../qa/workunits/ceph-helpers.sh:867: get_num_active_clean:  expression+=')'
       .../qa/workunits/ceph-helpers.sh:871: get_num_active_clean:  ceph --format xml pg dump pgs
       .../qa/workunits/ceph-helpers.sh:872: get_num_active_clean:  xmlstarlet sel -t -m '//pg_stat/state[(contains(.,'\''active'\'') and contains(.,'\''clean'\'') and not(contains(.,'\''stale'\'')))]' -v . -n
       .../qa/workunits/ceph-helpers.sh:873: get_num_active_clean:  grep -v '^$'
       .../qa/workunits/ceph-helpers.sh:873: get_num_active_clean:  wc -l
       ../qa/workunits/ceph-helpers.sh:993: wait_for_clean:  cur_active_clean=0
       ../qa/workunits/ceph-helpers.sh:994: wait_for_clean:  test 0 '!=' 0
       ../qa/workunits/ceph-helpers.sh:998: wait_for_clean:  sleep 1
       .../qa/workunits/ceph-helpers.sh:999: wait_for_clean:  expr 54 + 1
       ../qa/workunits/ceph-helpers.sh:999: wait_for_clean:  timer=55
       ../qa/workunits/ceph-helpers.sh:985: wait_for_clean:  is_clean
       .../qa/workunits/ceph-helpers.sh:952: is_clean:  get_num_pgs
       .../qa/workunits/ceph-helpers.sh:898: get_num_pgs:  ceph --format xml status
       .../qa/workunits/ceph-helpers.sh:899: get_num_pgs:  xmlstarlet sel -t -m //pgmap/num_pgs -v .
       ../qa/workunits/ceph-helpers.sh:952: is_clean:  num_pgs=4
       ../qa/workunits/ceph-helpers.sh:953: is_clean:  test 4 '!=' 0
       .../qa/workunits/ceph-helpers.sh:954: is_clean:  get_num_active_clean
       .../qa/workunits/ceph-helpers.sh:863: get_num_active_clean:  local 'expression=('
       .../qa/workunits/ceph-helpers.sh:864: get_num_active_clean:  expression+='contains(.,'\''active'\'') and '
       .../qa/workunits/ceph-helpers.sh:865: get_num_active_clean:  expression+='contains(.,'\''clean'\'') and '
       .../qa/workunits/ceph-helpers.sh:866: get_num_active_clean:  expression+='not(contains(.,'\''stale'\''))'
       .../qa/workunits/ceph-helpers.sh:867: get_num_active_clean:  expression+=')'
       .../qa/workunits/ceph-helpers.sh:871: get_num_active_clean:  ceph --format xml pg dump pgs
       .../qa/workunits/ceph-helpers.sh:872: get_num_active_clean:  xmlstarlet sel -t -m '//pg_stat/state[(contains(.,'\''active'\'') and contains(.,'\''clean'\'') and not(contains(.,'\''stale'\'')))]' -v . -n
       .../qa/workunits/ceph-helpers.sh:873: get_num_active_clean:  grep -v '^$'
       .../qa/workunits/ceph-helpers.sh:873: get_num_active_clean:  wc -l
       ../qa/workunits/ceph-helpers.sh:954: is_clean:  test 0 = 4
       ../qa/workunits/ceph-helpers.sh:954: is_clean:  return 1
       ../qa/workunits/ceph-helpers.sh:986: wait_for_clean:  get_is_making_recovery_progress
       .../qa/workunits/ceph-helpers.sh:839: get_is_making_recovery_progress:  ceph --format xml status
       .../qa/workunits/ceph-helpers.sh:839: get_is_making_recovery_progress:  xmlstarlet sel -t -m //pgmap/recovering_keys_per_sec -v . -o ' ' -t -m //pgmap/recovering_bytes_per_sec -v . -o ' ' -t -m //pgmap/recovering_objects_per_sec -v .
       ../qa/workunits/ceph-helpers.sh:839: get_is_making_recovery_progress:  local progress=
       ../qa/workunits/ceph-helpers.sh:840: get_is_making_recovery_progress:  test -n ''
       ../qa/workunits/ceph-helpers.sh:988: wait_for_clean:  ((  timer >= 120  ))
       .../qa/workunits/ceph-helpers.sh:993: wait_for_clean:  get_num_active_clean
       .../qa/workunits/ceph-helpers.sh:863: get_num_active_clean:  local 'expression=('
       .../qa/workunits/ceph-helpers.sh:864: get_num_active_clean:  expression+='contains(.,'\''active'\'') and '
       .../qa/workunits/ceph-helpers.sh:865: get_num_active_clean:  expression+='contains(.,'\''clean'\'') and '
       .../qa/workunits/ceph-helpers.sh:866: get_num_active_clean:  expression+='not(contains(.,'\''stale'\''))'
       .../qa/workunits/ceph-helpers.sh:867: get_num_active_clean:  expression+=')'
       .../qa/workunits/ceph-helpers.sh:871: get_num_active_clean:  ceph --format xml pg dump pgs
       .../qa/workunits/ceph-helpers.sh:872: get_num_active_clean:  xmlstarlet sel -t -m '//pg_stat/state[(contains(.,'\''active'\'') and contains(.,'\''clean'\'') and not(contains(.,'\''stale'\'')))]' -v . -n
       .../qa/workunits/ceph-helpers.sh:873: get_num_active_clean:  grep -v '^$'
       .../qa/workunits/ceph-helpers.sh:873: get_num_active_clean:  wc -l
       ../qa/workunits/ceph-helpers.sh:993: wait_for_clean:  cur_active_clean=0
       ../qa/workunits/ceph-helpers.sh:994: wait_for_clean:  test 0 '!=' 0
       ../qa/workunits/ceph-helpers.sh:998: wait_for_clean:  sleep 1
       .../qa/workunits/ceph-helpers.sh:999: wait_for_clean:  expr 55 + 1
       ../qa/workunits/ceph-helpers.sh:999: wait_for_clean:  timer=56
       ../qa/workunits/ceph-helpers.sh:985: wait_for_clean:  is_clean
       .../qa/workunits/ceph-helpers.sh:952: is_clean:  get_num_pgs
       .../qa/workunits/ceph-helpers.sh:898: get_num_pgs:  ceph --format xml status
       .../qa/workunits/ceph-helpers.sh:899: get_num_pgs:  xmlstarlet sel -t -m //pgmap/num_pgs -v .
       ../qa/workunits/ceph-helpers.sh:952: is_clean:  num_pgs=4
       ../qa/workunits/ceph-helpers.sh:953: is_clean:  test 4 '!=' 0
       .../qa/workunits/ceph-helpers.sh:954: is_clean:  get_num_active_clean
       .../qa/workunits/ceph-helpers.sh:863: get_num_active_clean:  local 'expression=('
       .../qa/workunits/ceph-helpers.sh:864: get_num_active_clean:  expression+='contains(.,'\''active'\'') and '
       .../qa/workunits/ceph-helpers.sh:865: get_num_active_clean:  expression+='contains(.,'\''clean'\'') and '
       .../qa/workunits/ceph-helpers.sh:866: get_num_active_clean:  expression+='not(contains(.,'\''stale'\''))'
       .../qa/workunits/ceph-helpers.sh:867: get_num_active_clean:  expression+=')'
       .../qa/workunits/ceph-helpers.sh:871: get_num_active_clean:  ceph --format xml pg dump pgs
       .../qa/workunits/ceph-helpers.sh:872: get_num_active_clean:  xmlstarlet sel -t -m '//pg_stat/state[(contains(.,'\''active'\'') and contains(.,'\''clean'\'') and not(contains(.,'\''stale'\'')))]' -v . -n
       .../qa/workunits/ceph-helpers.sh:873: get_num_active_clean:  wc -l
       .../qa/workunits/ceph-helpers.sh:873: get_num_active_clean:  grep -v '^$'
       ../qa/workunits/ceph-helpers.sh:954: is_clean:  test 0 = 4
       ../qa/workunits/ceph-helpers.sh:954: is_clean:  return 1
       ../qa/workunits/ceph-helpers.sh:986: wait_for_clean:  get_is_making_recovery_progress
       .../qa/workunits/ceph-helpers.sh:839: get_is_making_recovery_progress:  ceph --format xml status
       .../qa/workunits/ceph-helpers.sh:839: get_is_making_recovery_progress:  xmlstarlet sel -t -m //pgmap/recovering_keys_per_sec -v . -o ' ' -t -m //pgmap/recovering_bytes_per_sec -v . -o ' ' -t -m //pgmap/recovering_objects_per_sec -v .
       ../qa/workunits/ceph-helpers.sh:839: get_is_making_recovery_progress:  local progress=
       ../qa/workunits/ceph-helpers.sh:840: get_is_making_recovery_progress:  test -n ''
       ../qa/workunits/ceph-helpers.sh:988: wait_for_clean:  ((  timer >= 120  ))
       .../qa/workunits/ceph-helpers.sh:993: wait_for_clean:  get_num_active_clean
       .../qa/workunits/ceph-helpers.sh:863: get_num_active_clean:  local 'expression=('
       .../qa/workunits/ceph-helpers.sh:864: get_num_active_clean:  expression+='contains(.,'\''active'\'') and '
       .../qa/workunits/ceph-helpers.sh:865: get_num_active_clean:  expression+='contains(.,'\''clean'\'') and '
       .../qa/workunits/ceph-helpers.sh:866: get_num_active_clean:  expression+='not(contains(.,'\''stale'\''))'
       .../qa/workunits/ceph-helpers.sh:867: get_num_active_clean:  expression+=')'
       .../qa/workunits/ceph-helpers.sh:871: get_num_active_clean:  ceph --format xml pg dump pgs
       .../qa/workunits/ceph-helpers.sh:872: get_num_active_clean:  xmlstarlet sel -t -m '//pg_stat/state[(contains(.,'\''active'\'') and contains(.,'\''clean'\'') and not(contains(.,'\''stale'\'')))]' -v . -n
       .../qa/workunits/ceph-helpers.sh:873: get_num_active_clean:  grep -v '^$'
       .../qa/workunits/ceph-helpers.sh:873: get_num_active_clean:  wc -l
       ../qa/workunits/ceph-helpers.sh:993: wait_for_clean:  cur_active_clean=0
       ../qa/workunits/ceph-helpers.sh:994: wait_for_clean:  test 0 '!=' 0
       ../qa/workunits/ceph-helpers.sh:998: wait_for_clean:  sleep 1
       .../qa/workunits/ceph-helpers.sh:999: wait_for_clean:  expr 56 + 1
       ../qa/workunits/ceph-helpers.sh:999: wait_for_clean:  timer=57
       ../qa/workunits/ceph-helpers.sh:985: wait_for_clean:  is_clean
       .../qa/workunits/ceph-helpers.sh:952: is_clean:  get_num_pgs
       .../qa/workunits/ceph-helpers.sh:898: get_num_pgs:  ceph --format xml status
       .../qa/workunits/ceph-helpers.sh:899: get_num_pgs:  xmlstarlet sel -t -m //pgmap/num_pgs -v .
       ../qa/workunits/ceph-helpers.sh:952: is_clean:  num_pgs=4
       ../qa/workunits/ceph-helpers.sh:953: is_clean:  test 4 '!=' 0
       .../qa/workunits/ceph-helpers.sh:954: is_clean:  get_num_active_clean
       .../qa/workunits/ceph-helpers.sh:863: get_num_active_clean:  local 'expression=('
       .../qa/workunits/ceph-helpers.sh:864: get_num_active_clean:  expression+='contains(.,'\''active'\'') and '
       .../qa/workunits/ceph-helpers.sh:865: get_num_active_clean:  expression+='contains(.,'\''clean'\'') and '
       .../qa/workunits/ceph-helpers.sh:866: get_num_active_clean:  expression+='not(contains(.,'\''stale'\''))'
       .../qa/workunits/ceph-helpers.sh:867: get_num_active_clean:  expression+=')'
       .../qa/workunits/ceph-helpers.sh:871: get_num_active_clean:  ceph --format xml pg dump pgs
       .../qa/workunits/ceph-helpers.sh:872: get_num_active_clean:  xmlstarlet sel -t -m '//pg_stat/state[(contains(.,'\''active'\'') and contains(.,'\''clean'\'') and not(contains(.,'\''stale'\'')))]' -v . -n
       .../qa/workunits/ceph-helpers.sh:873: get_num_active_clean:  grep -v '^$'
       .../qa/workunits/ceph-helpers.sh:873: get_num_active_clean:  wc -l
       ../qa/workunits/ceph-helpers.sh:954: is_clean:  test 0 = 4
       ../qa/workunits/ceph-helpers.sh:954: is_clean:  return 1
       ../qa/workunits/ceph-helpers.sh:986: wait_for_clean:  get_is_making_recovery_progress
       .../qa/workunits/ceph-helpers.sh:839: get_is_making_recovery_progress:  ceph --format xml status
       .../qa/workunits/ceph-helpers.sh:839: get_is_making_recovery_progress:  xmlstarlet sel -t -m //pgmap/recovering_keys_per_sec -v . -o ' ' -t -m //pgmap/recovering_bytes_per_sec -v . -o ' ' -t -m //pgmap/recovering_objects_per_sec -v .
       ../qa/workunits/ceph-helpers.sh:839: get_is_making_recovery_progress:  local progress=
       ../qa/workunits/ceph-helpers.sh:840: get_is_making_recovery_progress:  test -n ''
       ../qa/workunits/ceph-helpers.sh:988: wait_for_clean:  ((  timer >= 120  ))
       .../qa/workunits/ceph-helpers.sh:993: wait_for_clean:  get_num_active_clean
       .../qa/workunits/ceph-helpers.sh:863: get_num_active_clean:  local 'expression=('
       .../qa/workunits/ceph-helpers.sh:864: get_num_active_clean:  expression+='contains(.,'\''active'\'') and '
       .../qa/workunits/ceph-helpers.sh:865: get_num_active_clean:  expression+='contains(.,'\''clean'\'') and '
       .../qa/workunits/ceph-helpers.sh:866: get_num_active_clean:  expression+='not(contains(.,'\''stale'\''))'
       .../qa/workunits/ceph-helpers.sh:867: get_num_active_clean:  expression+=')'
       .../qa/workunits/ceph-helpers.sh:871: get_num_active_clean:  ceph --format xml pg dump pgs
       .../qa/workunits/ceph-helpers.sh:872: get_num_active_clean:  xmlstarlet sel -t -m '//pg_stat/state[(contains(.,'\''active'\'') and contains(.,'\''clean'\'') and not(contains(.,'\''stale'\'')))]' -v . -n
       .../qa/workunits/ceph-helpers.sh:873: get_num_active_clean:  grep -v '^$'
       .../qa/workunits/ceph-helpers.sh:873: get_num_active_clean:  wc -l
       ../qa/workunits/ceph-helpers.sh:993: wait_for_clean:  cur_active_clean=0
       ../qa/workunits/ceph-helpers.sh:994: wait_for_clean:  test 0 '!=' 0
       ../qa/workunits/ceph-helpers.sh:998: wait_for_clean:  sleep 1
       .../qa/workunits/ceph-helpers.sh:999: wait_for_clean:  expr 57 + 1
       ../qa/workunits/ceph-helpers.sh:999: wait_for_clean:  timer=58
       ../qa/workunits/ceph-helpers.sh:985: wait_for_clean:  is_clean
       .../qa/workunits/ceph-helpers.sh:952: is_clean:  get_num_pgs
       .../qa/workunits/ceph-helpers.sh:898: get_num_pgs:  ceph --format xml status
       .../qa/workunits/ceph-helpers.sh:899: get_num_pgs:  xmlstarlet sel -t -m //pgmap/num_pgs -v .
       ../qa/workunits/ceph-helpers.sh:952: is_clean:  num_pgs=4
       ../qa/workunits/ceph-helpers.sh:953: is_clean:  test 4 '!=' 0
       .../qa/workunits/ceph-helpers.sh:954: is_clean:  get_num_active_clean
       .../qa/workunits/ceph-helpers.sh:863: get_num_active_clean:  local 'expression=('
       .../qa/workunits/ceph-helpers.sh:864: get_num_active_clean:  expression+='contains(.,'\''active'\'') and '
       .../qa/workunits/ceph-helpers.sh:865: get_num_active_clean:  expression+='contains(.,'\''clean'\'') and '
       .../qa/workunits/ceph-helpers.sh:866: get_num_active_clean:  expression+='not(contains(.,'\''stale'\''))'
       .../qa/workunits/ceph-helpers.sh:867: get_num_active_clean:  expression+=')'
       .../qa/workunits/ceph-helpers.sh:871: get_num_active_clean:  ceph --format xml pg dump pgs
       .../qa/workunits/ceph-helpers.sh:872: get_num_active_clean:  xmlstarlet sel -t -m '//pg_stat/state[(contains(.,'\''active'\'') and contains(.,'\''clean'\'') and not(contains(.,'\''stale'\'')))]' -v . -n
       .../qa/workunits/ceph-helpers.sh:873: get_num_active_clean:  wc -l
       .../qa/workunits/ceph-helpers.sh:873: get_num_active_clean:  grep -v '^$'
       ../qa/workunits/ceph-helpers.sh:954: is_clean:  test 0 = 4
       ../qa/workunits/ceph-helpers.sh:954: is_clean:  return 1
       ../qa/workunits/ceph-helpers.sh:986: wait_for_clean:  get_is_making_recovery_progress
       .../qa/workunits/ceph-helpers.sh:839: get_is_making_recovery_progress:  ceph --format xml status
       .../qa/workunits/ceph-helpers.sh:839: get_is_making_recovery_progress:  xmlstarlet sel -t -m //pgmap/recovering_keys_per_sec -v . -o ' ' -t -m //pgmap/recovering_bytes_per_sec -v . -o ' ' -t -m //pgmap/recovering_objects_per_sec -v .
       ../qa/workunits/ceph-helpers.sh:839: get_is_making_recovery_progress:  local progress=
       ../qa/workunits/ceph-helpers.sh:840: get_is_making_recovery_progress:  test -n ''
       ../qa/workunits/ceph-helpers.sh:988: wait_for_clean:  ((  timer >= 120  ))
       .../qa/workunits/ceph-helpers.sh:993: wait_for_clean:  get_num_active_clean
       .../qa/workunits/ceph-helpers.sh:863: get_num_active_clean:  local 'expression=('
       .../qa/workunits/ceph-helpers.sh:864: get_num_active_clean:  expression+='contains(.,'\''active'\'') and '
       .../qa/workunits/ceph-helpers.sh:865: get_num_active_clean:  expression+='contains(.,'\''clean'\'') and '
       .../qa/workunits/ceph-helpers.sh:866: get_num_active_clean:  expression+='not(contains(.,'\''stale'\''))'
       .../qa/workunits/ceph-helpers.sh:867: get_num_active_clean:  expression+=')'
       .../qa/workunits/ceph-helpers.sh:871: get_num_active_clean:  ceph --format xml pg dump pgs
       .../qa/workunits/ceph-helpers.sh:872: get_num_active_clean:  xmlstarlet sel -t -m '//pg_stat/state[(contains(.,'\''active'\'') and contains(.,'\''clean'\'') and not(contains(.,'\''stale'\'')))]' -v . -n
       .../qa/workunits/ceph-helpers.sh:873: get_num_active_clean:  grep -v '^$'
       .../qa/workunits/ceph-helpers.sh:873: get_num_active_clean:  wc -l
       ../qa/workunits/ceph-helpers.sh:993: wait_for_clean:  cur_active_clean=0
       ../qa/workunits/ceph-helpers.sh:994: wait_for_clean:  test 0 '!=' 0
       ../qa/workunits/ceph-helpers.sh:998: wait_for_clean:  sleep 1
       .../qa/workunits/ceph-helpers.sh:999: wait_for_clean:  expr 58 + 1
       ../qa/workunits/ceph-helpers.sh:999: wait_for_clean:  timer=59
       ../qa/workunits/ceph-helpers.sh:985: wait_for_clean:  is_clean
       .../qa/workunits/ceph-helpers.sh:952: is_clean:  get_num_pgs
       .../qa/workunits/ceph-helpers.sh:898: get_num_pgs:  ceph --format xml status
       .../qa/workunits/ceph-helpers.sh:899: get_num_pgs:  xmlstarlet sel -t -m //pgmap/num_pgs -v .
       ../qa/workunits/ceph-helpers.sh:952: is_clean:  num_pgs=4
       ../qa/workunits/ceph-helpers.sh:953: is_clean:  test 4 '!=' 0
       .../qa/workunits/ceph-helpers.sh:954: is_clean:  get_num_active_clean
       .../qa/workunits/ceph-helpers.sh:863: get_num_active_clean:  local 'expression=('
       .../qa/workunits/ceph-helpers.sh:864: get_num_active_clean:  expression+='contains(.,'\''active'\'') and '
       .../qa/workunits/ceph-helpers.sh:865: get_num_active_clean:  expression+='contains(.,'\''clean'\'') and '
       .../qa/workunits/ceph-helpers.sh:866: get_num_active_clean:  expression+='not(contains(.,'\''stale'\''))'
       .../qa/workunits/ceph-helpers.sh:867: get_num_active_clean:  expression+=')'
       .../qa/workunits/ceph-helpers.sh:871: get_num_active_clean:  ceph --format xml pg dump pgs
       .../qa/workunits/ceph-helpers.sh:872: get_num_active_clean:  xmlstarlet sel -t -m '//pg_stat/state[(contains(.,'\''active'\'') and contains(.,'\''clean'\'') and not(contains(.,'\''stale'\'')))]' -v . -n
       .../qa/workunits/ceph-helpers.sh:873: get_num_active_clean:  grep -v '^$'
       .../qa/workunits/ceph-helpers.sh:873: get_num_active_clean:  wc -l
       ../qa/workunits/ceph-helpers.sh:954: is_clean:  test 0 = 4
       ../qa/workunits/ceph-helpers.sh:954: is_clean:  return 1
       ../qa/workunits/ceph-helpers.sh:986: wait_for_clean:  get_is_making_recovery_progress
       .../qa/workunits/ceph-helpers.sh:839: get_is_making_recovery_progress:  ceph --format xml status
       .../qa/workunits/ceph-helpers.sh:839: get_is_making_recovery_progress:  xmlstarlet sel -t -m //pgmap/recovering_keys_per_sec -v . -o ' ' -t -m //pgmap/recovering_bytes_per_sec -v . -o ' ' -t -m //pgmap/recovering_objects_per_sec -v .
       ../qa/workunits/ceph-helpers.sh:839: get_is_making_recovery_progress:  local progress=
       ../qa/workunits/ceph-helpers.sh:840: get_is_making_recovery_progress:  test -n ''
       ../qa/workunits/ceph-helpers.sh:988: wait_for_clean:  ((  timer >= 120  ))
       .../qa/workunits/ceph-helpers.sh:993: wait_for_clean:  get_num_active_clean
       .../qa/workunits/ceph-helpers.sh:863: get_num_active_clean:  local 'expression=('
       .../qa/workunits/ceph-helpers.sh:864: get_num_active_clean:  expression+='contains(.,'\''active'\'') and '
       .../qa/workunits/ceph-helpers.sh:865: get_num_active_clean:  expression+='contains(.,'\''clean'\'') and '
       .../qa/workunits/ceph-helpers.sh:866: get_num_active_clean:  expression+='not(contains(.,'\''stale'\''))'
       .../qa/workunits/ceph-helpers.sh:867: get_num_active_clean:  expression+=')'
       .../qa/workunits/ceph-helpers.sh:871: get_num_active_clean:  ceph --format xml pg dump pgs
       .../qa/workunits/ceph-helpers.sh:872: get_num_active_clean:  xmlstarlet sel -t -m '//pg_stat/state[(contains(.,'\''active'\'') and contains(.,'\''clean'\'') and not(contains(.,'\''stale'\'')))]' -v . -n
       .../qa/workunits/ceph-helpers.sh:873: get_num_active_clean:  grep -v '^$'
       .../qa/workunits/ceph-helpers.sh:873: get_num_active_clean:  wc -l
       ../qa/workunits/ceph-helpers.sh:993: wait_for_clean:  cur_active_clean=0
       ../qa/workunits/ceph-helpers.sh:994: wait_for_clean:  test 0 '!=' 0
       ../qa/workunits/ceph-helpers.sh:998: wait_for_clean:  sleep 1
       .../qa/workunits/ceph-helpers.sh:999: wait_for_clean:  expr 59 + 1
       ../qa/workunits/ceph-helpers.sh:999: wait_for_clean:  timer=60
       ../qa/workunits/ceph-helpers.sh:985: wait_for_clean:  is_clean
       .../qa/workunits/ceph-helpers.sh:952: is_clean:  get_num_pgs
       .../qa/workunits/ceph-helpers.sh:898: get_num_pgs:  ceph --format xml status
       .../qa/workunits/ceph-helpers.sh:899: get_num_pgs:  xmlstarlet sel -t -m //pgmap/num_pgs -v .
       ../qa/workunits/ceph-helpers.sh:952: is_clean:  num_pgs=4
       ../qa/workunits/ceph-helpers.sh:953: is_clean:  test 4 '!=' 0
       .../qa/workunits/ceph-helpers.sh:954: is_clean:  get_num_active_clean
       .../qa/workunits/ceph-helpers.sh:863: get_num_active_clean:  local 'expression=('
       .../qa/workunits/ceph-helpers.sh:864: get_num_active_clean:  expression+='contains(.,'\''active'\'') and '
       .../qa/workunits/ceph-helpers.sh:865: get_num_active_clean:  expression+='contains(.,'\''clean'\'') and '
       .../qa/workunits/ceph-helpers.sh:866: get_num_active_clean:  expression+='not(contains(.,'\''stale'\''))'
       .../qa/workunits/ceph-helpers.sh:867: get_num_active_clean:  expression+=')'
       .../qa/workunits/ceph-helpers.sh:871: get_num_active_clean:  ceph --format xml pg dump pgs
       .../qa/workunits/ceph-helpers.sh:872: get_num_active_clean:  xmlstarlet sel -t -m '//pg_stat/state[(contains(.,'\''active'\'') and contains(.,'\''clean'\'') and not(contains(.,'\''stale'\'')))]' -v . -n
       .../qa/workunits/ceph-helpers.sh:873: get_num_active_clean:  wc -l
       .../qa/workunits/ceph-helpers.sh:873: get_num_active_clean:  grep -v '^$'
       ../qa/workunits/ceph-helpers.sh:954: is_clean:  test 0 = 4
       ../qa/workunits/ceph-helpers.sh:954: is_clean:  return 1
       ../qa/workunits/ceph-helpers.sh:986: wait_for_clean:  get_is_making_recovery_progress
       .../qa/workunits/ceph-helpers.sh:839: get_is_making_recovery_progress:  ceph --format xml status
       .../qa/workunits/ceph-helpers.sh:839: get_is_making_recovery_progress:  xmlstarlet sel -t -m //pgmap/recovering_keys_per_sec -v . -o ' ' -t -m //pgmap/recovering_bytes_per_sec -v . -o ' ' -t -m //pgmap/recovering_objects_per_sec -v .
       ../qa/workunits/ceph-helpers.sh:839: get_is_making_recovery_progress:  local progress=
       ../qa/workunits/ceph-helpers.sh:840: get_is_making_recovery_progress:  test -n ''
       ../qa/workunits/ceph-helpers.sh:988: wait_for_clean:  ((  timer >= 120  ))
       .../qa/workunits/ceph-helpers.sh:993: wait_for_clean:  get_num_active_clean
       .../qa/workunits/ceph-helpers.sh:863: get_num_active_clean:  local 'expression=('
       .../qa/workunits/ceph-helpers.sh:864: get_num_active_clean:  expression+='contains(.,'\''active'\'') and '
       .../qa/workunits/ceph-helpers.sh:865: get_num_active_clean:  expression+='contains(.,'\''clean'\'') and '
       .../qa/workunits/ceph-helpers.sh:866: get_num_active_clean:  expression+='not(contains(.,'\''stale'\''))'
       .../qa/workunits/ceph-helpers.sh:867: get_num_active_clean:  expression+=')'
       .../qa/workunits/ceph-helpers.sh:871: get_num_active_clean:  ceph --format xml pg dump pgs
       .../qa/workunits/ceph-helpers.sh:872: get_num_active_clean:  xmlstarlet sel -t -m '//pg_stat/state[(contains(.,'\''active'\'') and contains(.,'\''clean'\'') and not(contains(.,'\''stale'\'')))]' -v . -n
       .../qa/workunits/ceph-helpers.sh:873: get_num_active_clean:  wc -l
       .../qa/workunits/ceph-helpers.sh:873: get_num_active_clean:  grep -v '^$'
       ../qa/workunits/ceph-helpers.sh:993: wait_for_clean:  cur_active_clean=0
       ../qa/workunits/ceph-helpers.sh:994: wait_for_clean:  test 0 '!=' 0
       ../qa/workunits/ceph-helpers.sh:998: wait_for_clean:  sleep 1
       .../qa/workunits/ceph-helpers.sh:999: wait_for_clean:  expr 60 + 1
       ../qa/workunits/ceph-helpers.sh:999: wait_for_clean:  timer=61
       ../qa/workunits/ceph-helpers.sh:985: wait_for_clean:  is_clean
       .../qa/workunits/ceph-helpers.sh:952: is_clean:  get_num_pgs
       .../qa/workunits/ceph-helpers.sh:898: get_num_pgs:  ceph --format xml status
       .../qa/workunits/ceph-helpers.sh:899: get_num_pgs:  xmlstarlet sel -t -m //pgmap/num_pgs -v .
       ../qa/workunits/ceph-helpers.sh:952: is_clean:  num_pgs=4
       ../qa/workunits/ceph-helpers.sh:953: is_clean:  test 4 '!=' 0
       .../qa/workunits/ceph-helpers.sh:954: is_clean:  get_num_active_clean
       .../qa/workunits/ceph-helpers.sh:863: get_num_active_clean:  local 'expression=('
       .../qa/workunits/ceph-helpers.sh:864: get_num_active_clean:  expression+='contains(.,'\''active'\'') and '
       .../qa/workunits/ceph-helpers.sh:865: get_num_active_clean:  expression+='contains(.,'\''clean'\'') and '
       .../qa/workunits/ceph-helpers.sh:866: get_num_active_clean:  expression+='not(contains(.,'\''stale'\''))'
       .../qa/workunits/ceph-helpers.sh:867: get_num_active_clean:  expression+=')'
       .../qa/workunits/ceph-helpers.sh:871: get_num_active_clean:  ceph --format xml pg dump pgs
       .../qa/workunits/ceph-helpers.sh:872: get_num_active_clean:  xmlstarlet sel -t -m '//pg_stat/state[(contains(.,'\''active'\'') and contains(.,'\''clean'\'') and not(contains(.,'\''stale'\'')))]' -v . -n
       .../qa/workunits/ceph-helpers.sh:873: get_num_active_clean:  grep -v '^$'
       .../qa/workunits/ceph-helpers.sh:873: get_num_active_clean:  wc -l
       ../qa/workunits/ceph-helpers.sh:954: is_clean:  test 0 = 4
       ../qa/workunits/ceph-helpers.sh:954: is_clean:  return 1
       ../qa/workunits/ceph-helpers.sh:986: wait_for_clean:  get_is_making_recovery_progress
       .../qa/workunits/ceph-helpers.sh:839: get_is_making_recovery_progress:  ceph --format xml status
       .../qa/workunits/ceph-helpers.sh:839: get_is_making_recovery_progress:  xmlstarlet sel -t -m //pgmap/recovering_keys_per_sec -v . -o ' ' -t -m //pgmap/recovering_bytes_per_sec -v . -o ' ' -t -m //pgmap/recovering_objects_per_sec -v .
       ../qa/workunits/ceph-helpers.sh:839: get_is_making_recovery_progress:  local progress=
       ../qa/workunits/ceph-helpers.sh:840: get_is_making_recovery_progress:  test -n ''
       ../qa/workunits/ceph-helpers.sh:988: wait_for_clean:  ((  timer >= 120  ))
       .../qa/workunits/ceph-helpers.sh:993: wait_for_clean:  get_num_active_clean
       .../qa/workunits/ceph-helpers.sh:863: get_num_active_clean:  local 'expression=('
       .../qa/workunits/ceph-helpers.sh:864: get_num_active_clean:  expression+='contains(.,'\''active'\'') and '
       .../qa/workunits/ceph-helpers.sh:865: get_num_active_clean:  expression+='contains(.,'\''clean'\'') and '
       .../qa/workunits/ceph-helpers.sh:866: get_num_active_clean:  expression+='not(contains(.,'\''stale'\''))'
       .../qa/workunits/ceph-helpers.sh:867: get_num_active_clean:  expression+=')'
       .../qa/workunits/ceph-helpers.sh:871: get_num_active_clean:  ceph --format xml pg dump pgs
       .../qa/workunits/ceph-helpers.sh:872: get_num_active_clean:  xmlstarlet sel -t -m '//pg_stat/state[(contains(.,'\''active'\'') and contains(.,'\''clean'\'') and not(contains(.,'\''stale'\'')))]' -v . -n
       .../qa/workunits/ceph-helpers.sh:873: get_num_active_clean:  grep -v '^$'
       .../qa/workunits/ceph-helpers.sh:873: get_num_active_clean:  wc -l
       ../qa/workunits/ceph-helpers.sh:993: wait_for_clean:  cur_active_clean=0
       ../qa/workunits/ceph-helpers.sh:994: wait_for_clean:  test 0 '!=' 0
       ../qa/workunits/ceph-helpers.sh:998: wait_for_clean:  sleep 1
       .../qa/workunits/ceph-helpers.sh:999: wait_for_clean:  expr 61 + 1
       ../qa/workunits/ceph-helpers.sh:999: wait_for_clean:  timer=62
       ../qa/workunits/ceph-helpers.sh:985: wait_for_clean:  is_clean
       .../qa/workunits/ceph-helpers.sh:952: is_clean:  get_num_pgs
       .../qa/workunits/ceph-helpers.sh:898: get_num_pgs:  ceph --format xml status
       .../qa/workunits/ceph-helpers.sh:899: get_num_pgs:  xmlstarlet sel -t -m //pgmap/num_pgs -v .
       ../qa/workunits/ceph-helpers.sh:952: is_clean:  num_pgs=4
       ../qa/workunits/ceph-helpers.sh:953: is_clean:  test 4 '!=' 0
       .../qa/workunits/ceph-helpers.sh:954: is_clean:  get_num_active_clean
       .../qa/workunits/ceph-helpers.sh:863: get_num_active_clean:  local 'expression=('
       .../qa/workunits/ceph-helpers.sh:864: get_num_active_clean:  expression+='contains(.,'\''active'\'') and '
       .../qa/workunits/ceph-helpers.sh:865: get_num_active_clean:  expression+='contains(.,'\''clean'\'') and '
       .../qa/workunits/ceph-helpers.sh:866: get_num_active_clean:  expression+='not(contains(.,'\''stale'\''))'
       .../qa/workunits/ceph-helpers.sh:867: get_num_active_clean:  expression+=')'
       .../qa/workunits/ceph-helpers.sh:871: get_num_active_clean:  ceph --format xml pg dump pgs
       .../qa/workunits/ceph-helpers.sh:873: get_num_active_clean:  grep -v '^$'
       .../qa/workunits/ceph-helpers.sh:872: get_num_active_clean:  xmlstarlet sel -t -m '//pg_stat/state[(contains(.,'\''active'\'') and contains(.,'\''clean'\'') and not(contains(.,'\''stale'\'')))]' -v . -n
       .../qa/workunits/ceph-helpers.sh:873: get_num_active_clean:  wc -l
       ../qa/workunits/ceph-helpers.sh:954: is_clean:  test 0 = 4
       ../qa/workunits/ceph-helpers.sh:954: is_clean:  return 1
       ../qa/workunits/ceph-helpers.sh:986: wait_for_clean:  get_is_making_recovery_progress
       .../qa/workunits/ceph-helpers.sh:839: get_is_making_recovery_progress:  ceph --format xml status
       .../qa/workunits/ceph-helpers.sh:839: get_is_making_recovery_progress:  xmlstarlet sel -t -m //pgmap/recovering_keys_per_sec -v . -o ' ' -t -m //pgmap/recovering_bytes_per_sec -v . -o ' ' -t -m //pgmap/recovering_objects_per_sec -v .
       ../qa/workunits/ceph-helpers.sh:839: get_is_making_recovery_progress:  local progress=
       ../qa/workunits/ceph-helpers.sh:840: get_is_making_recovery_progress:  test -n ''
       ../qa/workunits/ceph-helpers.sh:988: wait_for_clean:  ((  timer >= 120  ))
       .../qa/workunits/ceph-helpers.sh:993: wait_for_clean:  get_num_active_clean
       .../qa/workunits/ceph-helpers.sh:863: get_num_active_clean:  local 'expression=('
       .../qa/workunits/ceph-helpers.sh:864: get_num_active_clean:  expression+='contains(.,'\''active'\'') and '
       .../qa/workunits/ceph-helpers.sh:865: get_num_active_clean:  expression+='contains(.,'\''clean'\'') and '
       .../qa/workunits/ceph-helpers.sh:866: get_num_active_clean:  expression+='not(contains(.,'\''stale'\''))'
       .../qa/workunits/ceph-helpers.sh:867: get_num_active_clean:  expression+=')'
       .../qa/workunits/ceph-helpers.sh:871: get_num_active_clean:  ceph --format xml pg dump pgs
       .../qa/workunits/ceph-helpers.sh:872: get_num_active_clean:  xmlstarlet sel -t -m '//pg_stat/state[(contains(.,'\''active'\'') and contains(.,'\''clean'\'') and not(contains(.,'\''stale'\'')))]' -v . -n
       .../qa/workunits/ceph-helpers.sh:873: get_num_active_clean:  grep -v '^$'
       .../qa/workunits/ceph-helpers.sh:873: get_num_active_clean:  wc -l
       ../qa/workunits/ceph-helpers.sh:993: wait_for_clean:  cur_active_clean=0
       ../qa/workunits/ceph-helpers.sh:994: wait_for_clean:  test 0 '!=' 0
       ../qa/workunits/ceph-helpers.sh:998: wait_for_clean:  sleep 1
       .../qa/workunits/ceph-helpers.sh:999: wait_for_clean:  expr 62 + 1
       ../qa/workunits/ceph-helpers.sh:999: wait_for_clean:  timer=63
       ../qa/workunits/ceph-helpers.sh:985: wait_for_clean:  is_clean
       .../qa/workunits/ceph-helpers.sh:952: is_clean:  get_num_pgs
       .../qa/workunits/ceph-helpers.sh:898: get_num_pgs:  ceph --format xml status
       .../qa/workunits/ceph-helpers.sh:899: get_num_pgs:  xmlstarlet sel -t -m //pgmap/num_pgs -v .
       ../qa/workunits/ceph-helpers.sh:952: is_clean:  num_pgs=4
       ../qa/workunits/ceph-helpers.sh:953: is_clean:  test 4 '!=' 0
       .../qa/workunits/ceph-helpers.sh:954: is_clean:  get_num_active_clean
       .../qa/workunits/ceph-helpers.sh:863: get_num_active_clean:  local 'expression=('
       .../qa/workunits/ceph-helpers.sh:864: get_num_active_clean:  expression+='contains(.,'\''active'\'') and '
       .../qa/workunits/ceph-helpers.sh:865: get_num_active_clean:  expression+='contains(.,'\''clean'\'') and '
       .../qa/workunits/ceph-helpers.sh:866: get_num_active_clean:  expression+='not(contains(.,'\''stale'\''))'
       .../qa/workunits/ceph-helpers.sh:867: get_num_active_clean:  expression+=')'
       .../qa/workunits/ceph-helpers.sh:871: get_num_active_clean:  ceph --format xml pg dump pgs
       .../qa/workunits/ceph-helpers.sh:872: get_num_active_clean:  xmlstarlet sel -t -m '//pg_stat/state[(contains(.,'\''active'\'') and contains(.,'\''clean'\'') and not(contains(.,'\''stale'\'')))]' -v . -n
       .../qa/workunits/ceph-helpers.sh:873: get_num_active_clean:  grep -v '^$'
       .../qa/workunits/ceph-helpers.sh:873: get_num_active_clean:  wc -l
       ../qa/workunits/ceph-helpers.sh:954: is_clean:  test 0 = 4
       ../qa/workunits/ceph-helpers.sh:954: is_clean:  return 1
       ../qa/workunits/ceph-helpers.sh:986: wait_for_clean:  get_is_making_recovery_progress
       .../qa/workunits/ceph-helpers.sh:839: get_is_making_recovery_progress:  ceph --format xml status
       .../qa/workunits/ceph-helpers.sh:839: get_is_making_recovery_progress:  xmlstarlet sel -t -m //pgmap/recovering_keys_per_sec -v . -o ' ' -t -m //pgmap/recovering_bytes_per_sec -v . -o ' ' -t -m //pgmap/recovering_objects_per_sec -v .
       ../qa/workunits/ceph-helpers.sh:839: get_is_making_recovery_progress:  local progress=
       ../qa/workunits/ceph-helpers.sh:840: get_is_making_recovery_progress:  test -n ''
       ../qa/workunits/ceph-helpers.sh:988: wait_for_clean:  ((  timer >= 120  ))
       .../qa/workunits/ceph-helpers.sh:993: wait_for_clean:  get_num_active_clean
       .../qa/workunits/ceph-helpers.sh:863: get_num_active_clean:  local 'expression=('
       .../qa/workunits/ceph-helpers.sh:864: get_num_active_clean:  expression+='contains(.,'\''active'\'') and '
       .../qa/workunits/ceph-helpers.sh:865: get_num_active_clean:  expression+='contains(.,'\''clean'\'') and '
       .../qa/workunits/ceph-helpers.sh:866: get_num_active_clean:  expression+='not(contains(.,'\''stale'\''))'
       .../qa/workunits/ceph-helpers.sh:867: get_num_active_clean:  expression+=')'
       .../qa/workunits/ceph-helpers.sh:871: get_num_active_clean:  ceph --format xml pg dump pgs
       .../qa/workunits/ceph-helpers.sh:872: get_num_active_clean:  xmlstarlet sel -t -m '//pg_stat/state[(contains(.,'\''active'\'') and contains(.,'\''clean'\'') and not(contains(.,'\''stale'\'')))]' -v . -n
       .../qa/workunits/ceph-helpers.sh:873: get_num_active_clean:  grep -v '^$'
       .../qa/workunits/ceph-helpers.sh:873: get_num_active_clean:  wc -l
       ../qa/workunits/ceph-helpers.sh:993: wait_for_clean:  cur_active_clean=0
       ../qa/workunits/ceph-helpers.sh:994: wait_for_clean:  test 0 '!=' 0
       ../qa/workunits/ceph-helpers.sh:998: wait_for_clean:  sleep 1
       .../qa/workunits/ceph-helpers.sh:999: wait_for_clean:  expr 63 + 1
       ../qa/workunits/ceph-helpers.sh:999: wait_for_clean:  timer=64
       ../qa/workunits/ceph-helpers.sh:985: wait_for_clean:  is_clean
       .../qa/workunits/ceph-helpers.sh:952: is_clean:  get_num_pgs
       .../qa/workunits/ceph-helpers.sh:898: get_num_pgs:  ceph --format xml status
       .../qa/workunits/ceph-helpers.sh:899: get_num_pgs:  xmlstarlet sel -t -m //pgmap/num_pgs -v .
       ../qa/workunits/ceph-helpers.sh:952: is_clean:  num_pgs=4
       ../qa/workunits/ceph-helpers.sh:953: is_clean:  test 4 '!=' 0
       .../qa/workunits/ceph-helpers.sh:954: is_clean:  get_num_active_clean
       .../qa/workunits/ceph-helpers.sh:863: get_num_active_clean:  local 'expression=('
       .../qa/workunits/ceph-helpers.sh:864: get_num_active_clean:  expression+='contains(.,'\''active'\'') and '
       .../qa/workunits/ceph-helpers.sh:865: get_num_active_clean:  expression+='contains(.,'\''clean'\'') and '
       .../qa/workunits/ceph-helpers.sh:866: get_num_active_clean:  expression+='not(contains(.,'\''stale'\''))'
       .../qa/workunits/ceph-helpers.sh:867: get_num_active_clean:  expression+=')'
       .../qa/workunits/ceph-helpers.sh:871: get_num_active_clean:  ceph --format xml pg dump pgs
       .../qa/workunits/ceph-helpers.sh:872: get_num_active_clean:  xmlstarlet sel -t -m '//pg_stat/state[(contains(.,'\''active'\'') and contains(.,'\''clean'\'') and not(contains(.,'\''stale'\'')))]' -v . -n
       .../qa/workunits/ceph-helpers.sh:873: get_num_active_clean:  grep -v '^$'
       .../qa/workunits/ceph-helpers.sh:873: get_num_active_clean:  wc -l
       ../qa/workunits/ceph-helpers.sh:954: is_clean:  test 0 = 4
       ../qa/workunits/ceph-helpers.sh:954: is_clean:  return 1
       ../qa/workunits/ceph-helpers.sh:986: wait_for_clean:  get_is_making_recovery_progress
       .../qa/workunits/ceph-helpers.sh:839: get_is_making_recovery_progress:  ceph --format xml status
       .../qa/workunits/ceph-helpers.sh:839: get_is_making_recovery_progress:  xmlstarlet sel -t -m //pgmap/recovering_keys_per_sec -v . -o ' ' -t -m //pgmap/recovering_bytes_per_sec -v . -o ' ' -t -m //pgmap/recovering_objects_per_sec -v .
       ../qa/workunits/ceph-helpers.sh:839: get_is_making_recovery_progress:  local progress=
       ../qa/workunits/ceph-helpers.sh:840: get_is_making_recovery_progress:  test -n ''
       ../qa/workunits/ceph-helpers.sh:988: wait_for_clean:  ((  timer >= 120  ))
       .../qa/workunits/ceph-helpers.sh:993: wait_for_clean:  get_num_active_clean
       .../qa/workunits/ceph-helpers.sh:863: get_num_active_clean:  local 'expression=('
       .../qa/workunits/ceph-helpers.sh:864: get_num_active_clean:  expression+='contains(.,'\''active'\'') and '
       .../qa/workunits/ceph-helpers.sh:865: get_num_active_clean:  expression+='contains(.,'\''clean'\'') and '
       .../qa/workunits/ceph-helpers.sh:866: get_num_active_clean:  expression+='not(contains(.,'\''stale'\''))'
       .../qa/workunits/ceph-helpers.sh:867: get_num_active_clean:  expression+=')'
       .../qa/workunits/ceph-helpers.sh:871: get_num_active_clean:  ceph --format xml pg dump pgs
       .../qa/workunits/ceph-helpers.sh:872: get_num_active_clean:  xmlstarlet sel -t -m '//pg_stat/state[(contains(.,'\''active'\'') and contains(.,'\''clean'\'') and not(contains(.,'\''stale'\'')))]' -v . -n
       .../qa/workunits/ceph-helpers.sh:873: get_num_active_clean:  wc -l
       .../qa/workunits/ceph-helpers.sh:873: get_num_active_clean:  grep -v '^$'
       ../qa/workunits/ceph-helpers.sh:993: wait_for_clean:  cur_active_clean=0
       ../qa/workunits/ceph-helpers.sh:994: wait_for_clean:  test 0 '!=' 0
       ../qa/workunits/ceph-helpers.sh:998: wait_for_clean:  sleep 1
       .../qa/workunits/ceph-helpers.sh:999: wait_for_clean:  expr 64 + 1
       ../qa/workunits/ceph-helpers.sh:999: wait_for_clean:  timer=65
       ../qa/workunits/ceph-helpers.sh:985: wait_for_clean:  is_clean
       .../qa/workunits/ceph-helpers.sh:952: is_clean:  get_num_pgs
       .../qa/workunits/ceph-helpers.sh:898: get_num_pgs:  ceph --format xml status
       .../qa/workunits/ceph-helpers.sh:899: get_num_pgs:  xmlstarlet sel -t -m //pgmap/num_pgs -v .
       ../qa/workunits/ceph-helpers.sh:952: is_clean:  num_pgs=4
       ../qa/workunits/ceph-helpers.sh:953: is_clean:  test 4 '!=' 0
       .../qa/workunits/ceph-helpers.sh:954: is_clean:  get_num_active_clean
       .../qa/workunits/ceph-helpers.sh:863: get_num_active_clean:  local 'expression=('
       .../qa/workunits/ceph-helpers.sh:864: get_num_active_clean:  expression+='contains(.,'\''active'\'') and '
       .../qa/workunits/ceph-helpers.sh:865: get_num_active_clean:  expression+='contains(.,'\''clean'\'') and '
       .../qa/workunits/ceph-helpers.sh:866: get_num_active_clean:  expression+='not(contains(.,'\''stale'\''))'
       .../qa/workunits/ceph-helpers.sh:867: get_num_active_clean:  expression+=')'
       .../qa/workunits/ceph-helpers.sh:871: get_num_active_clean:  ceph --format xml pg dump pgs
       .../qa/workunits/ceph-helpers.sh:872: get_num_active_clean:  xmlstarlet sel -t -m '//pg_stat/state[(contains(.,'\''active'\'') and contains(.,'\''clean'\'') and not(contains(.,'\''stale'\'')))]' -v . -n
       .../qa/workunits/ceph-helpers.sh:873: get_num_active_clean:  grep -v '^$'
       .../qa/workunits/ceph-helpers.sh:873: get_num_active_clean:  wc -l
       ../qa/workunits/ceph-helpers.sh:954: is_clean:  test 0 = 4
       ../qa/workunits/ceph-helpers.sh:954: is_clean:  return 1
       ../qa/workunits/ceph-helpers.sh:986: wait_for_clean:  get_is_making_recovery_progress
       .../qa/workunits/ceph-helpers.sh:839: get_is_making_recovery_progress:  ceph --format xml status
       .../qa/workunits/ceph-helpers.sh:839: get_is_making_recovery_progress:  xmlstarlet sel -t -m //pgmap/recovering_keys_per_sec -v . -o ' ' -t -m //pgmap/recovering_bytes_per_sec -v . -o ' ' -t -m //pgmap/recovering_objects_per_sec -v .
       ../qa/workunits/ceph-helpers.sh:839: get_is_making_recovery_progress:  local progress=
       ../qa/workunits/ceph-helpers.sh:840: get_is_making_recovery_progress:  test -n ''
       ../qa/workunits/ceph-helpers.sh:988: wait_for_clean:  ((  timer >= 120  ))
       .../qa/workunits/ceph-helpers.sh:993: wait_for_clean:  get_num_active_clean
       .../qa/workunits/ceph-helpers.sh:863: get_num_active_clean:  local 'expression=('
       .../qa/workunits/ceph-helpers.sh:864: get_num_active_clean:  expression+='contains(.,'\''active'\'') and '
       .../qa/workunits/ceph-helpers.sh:865: get_num_active_clean:  expression+='contains(.,'\''clean'\'') and '
       .../qa/workunits/ceph-helpers.sh:866: get_num_active_clean:  expression+='not(contains(.,'\''stale'\''))'
       .../qa/workunits/ceph-helpers.sh:867: get_num_active_clean:  expression+=')'
       .../qa/workunits/ceph-helpers.sh:871: get_num_active_clean:  ceph --format xml pg dump pgs
       .../qa/workunits/ceph-helpers.sh:872: get_num_active_clean:  xmlstarlet sel -t -m '//pg_stat/state[(contains(.,'\''active'\'') and contains(.,'\''clean'\'') and not(contains(.,'\''stale'\'')))]' -v . -n
       .../qa/workunits/ceph-helpers.sh:873: get_num_active_clean:  grep -v '^$'
       .../qa/workunits/ceph-helpers.sh:873: get_num_active_clean:  wc -l
       ../qa/workunits/ceph-helpers.sh:993: wait_for_clean:  cur_active_clean=0
       ../qa/workunits/ceph-helpers.sh:994: wait_for_clean:  test 0 '!=' 0
       ../qa/workunits/ceph-helpers.sh:998: wait_for_clean:  sleep 1
       .../qa/workunits/ceph-helpers.sh:999: wait_for_clean:  expr 65 + 1
       ../qa/workunits/ceph-helpers.sh:999: wait_for_clean:  timer=66
       ../qa/workunits/ceph-helpers.sh:985: wait_for_clean:  is_clean
       .../qa/workunits/ceph-helpers.sh:952: is_clean:  get_num_pgs
       .../qa/workunits/ceph-helpers.sh:898: get_num_pgs:  ceph --format xml status
       .../qa/workunits/ceph-helpers.sh:899: get_num_pgs:  xmlstarlet sel -t -m //pgmap/num_pgs -v .
       ../qa/workunits/ceph-helpers.sh:952: is_clean:  num_pgs=4
       ../qa/workunits/ceph-helpers.sh:953: is_clean:  test 4 '!=' 0
       .../qa/workunits/ceph-helpers.sh:954: is_clean:  get_num_active_clean
       .../qa/workunits/ceph-helpers.sh:863: get_num_active_clean:  local 'expression=('
       .../qa/workunits/ceph-helpers.sh:864: get_num_active_clean:  expression+='contains(.,'\''active'\'') and '
       .../qa/workunits/ceph-helpers.sh:865: get_num_active_clean:  expression+='contains(.,'\''clean'\'') and '
       .../qa/workunits/ceph-helpers.sh:866: get_num_active_clean:  expression+='not(contains(.,'\''stale'\''))'
       .../qa/workunits/ceph-helpers.sh:867: get_num_active_clean:  expression+=')'
       .../qa/workunits/ceph-helpers.sh:871: get_num_active_clean:  ceph --format xml pg dump pgs
       .../qa/workunits/ceph-helpers.sh:872: get_num_active_clean:  xmlstarlet sel -t -m '//pg_stat/state[(contains(.,'\''active'\'') and contains(.,'\''clean'\'') and not(contains(.,'\''stale'\'')))]' -v . -n
       .../qa/workunits/ceph-helpers.sh:873: get_num_active_clean:  grep -v '^$'
       .../qa/workunits/ceph-helpers.sh:873: get_num_active_clean:  wc -l
       ../qa/workunits/ceph-helpers.sh:954: is_clean:  test 0 = 4
       ../qa/workunits/ceph-helpers.sh:954: is_clean:  return 1
       ../qa/workunits/ceph-helpers.sh:986: wait_for_clean:  get_is_making_recovery_progress
       .../qa/workunits/ceph-helpers.sh:839: get_is_making_recovery_progress:  ceph --format xml status
       .../qa/workunits/ceph-helpers.sh:839: get_is_making_recovery_progress:  xmlstarlet sel -t -m //pgmap/recovering_keys_per_sec -v . -o ' ' -t -m //pgmap/recovering_bytes_per_sec -v . -o ' ' -t -m //pgmap/recovering_objects_per_sec -v .
       ../qa/workunits/ceph-helpers.sh:839: get_is_making_recovery_progress:  local progress=
       ../qa/workunits/ceph-helpers.sh:840: get_is_making_recovery_progress:  test -n ''
       ../qa/workunits/ceph-helpers.sh:988: wait_for_clean:  ((  timer >= 120  ))
       .../qa/workunits/ceph-helpers.sh:993: wait_for_clean:  get_num_active_clean
       .../qa/workunits/ceph-helpers.sh:863: get_num_active_clean:  local 'expression=('
       .../qa/workunits/ceph-helpers.sh:864: get_num_active_clean:  expression+='contains(.,'\''active'\'') and '
       .../qa/workunits/ceph-helpers.sh:865: get_num_active_clean:  expression+='contains(.,'\''clean'\'') and '
       .../qa/workunits/ceph-helpers.sh:866: get_num_active_clean:  expression+='not(contains(.,'\''stale'\''))'
       .../qa/workunits/ceph-helpers.sh:867: get_num_active_clean:  expression+=')'
       .../qa/workunits/ceph-helpers.sh:871: get_num_active_clean:  ceph --format xml pg dump pgs
       .../qa/workunits/ceph-helpers.sh:872: get_num_active_clean:  xmlstarlet sel -t -m '//pg_stat/state[(contains(.,'\''active'\'') and contains(.,'\''clean'\'') and not(contains(.,'\''stale'\'')))]' -v . -n
       .../qa/workunits/ceph-helpers.sh:873: get_num_active_clean:  grep -v '^$'
       .../qa/workunits/ceph-helpers.sh:873: get_num_active_clean:  wc -l
       ../qa/workunits/ceph-helpers.sh:993: wait_for_clean:  cur_active_clean=0
       ../qa/workunits/ceph-helpers.sh:994: wait_for_clean:  test 0 '!=' 0
       ../qa/workunits/ceph-helpers.sh:998: wait_for_clean:  sleep 1
       .../qa/workunits/ceph-helpers.sh:999: wait_for_clean:  expr 66 + 1
       ../qa/workunits/ceph-helpers.sh:999: wait_for_clean:  timer=67
       ../qa/workunits/ceph-helpers.sh:985: wait_for_clean:  is_clean
       .../qa/workunits/ceph-helpers.sh:952: is_clean:  get_num_pgs
       .../qa/workunits/ceph-helpers.sh:898: get_num_pgs:  ceph --format xml status
       .../qa/workunits/ceph-helpers.sh:899: get_num_pgs:  xmlstarlet sel -t -m //pgmap/num_pgs -v .
       ../qa/workunits/ceph-helpers.sh:952: is_clean:  num_pgs=4
       ../qa/workunits/ceph-helpers.sh:953: is_clean:  test 4 '!=' 0
       .../qa/workunits/ceph-helpers.sh:954: is_clean:  get_num_active_clean
       .../qa/workunits/ceph-helpers.sh:863: get_num_active_clean:  local 'expression=('
       .../qa/workunits/ceph-helpers.sh:864: get_num_active_clean:  expression+='contains(.,'\''active'\'') and '
       .../qa/workunits/ceph-helpers.sh:865: get_num_active_clean:  expression+='contains(.,'\''clean'\'') and '
       .../qa/workunits/ceph-helpers.sh:866: get_num_active_clean:  expression+='not(contains(.,'\''stale'\''))'
       .../qa/workunits/ceph-helpers.sh:867: get_num_active_clean:  expression+=')'
       .../qa/workunits/ceph-helpers.sh:871: get_num_active_clean:  ceph --format xml pg dump pgs
       .../qa/workunits/ceph-helpers.sh:872: get_num_active_clean:  xmlstarlet sel -t -m '//pg_stat/state[(contains(.,'\''active'\'') and contains(.,'\''clean'\'') and not(contains(.,'\''stale'\'')))]' -v . -n
       .../qa/workunits/ceph-helpers.sh:873: get_num_active_clean:  wc -l
       .../qa/workunits/ceph-helpers.sh:873: get_num_active_clean:  grep -v '^$'
       ../qa/workunits/ceph-helpers.sh:954: is_clean:  test 0 = 4
       ../qa/workunits/ceph-helpers.sh:954: is_clean:  return 1
       ../qa/workunits/ceph-helpers.sh:986: wait_for_clean:  get_is_making_recovery_progress
       .../qa/workunits/ceph-helpers.sh:839: get_is_making_recovery_progress:  ceph --format xml status
       .../qa/workunits/ceph-helpers.sh:839: get_is_making_recovery_progress:  xmlstarlet sel -t -m //pgmap/recovering_keys_per_sec -v . -o ' ' -t -m //pgmap/recovering_bytes_per_sec -v . -o ' ' -t -m //pgmap/recovering_objects_per_sec -v .
       ../qa/workunits/ceph-helpers.sh:839: get_is_making_recovery_progress:  local progress=
       ../qa/workunits/ceph-helpers.sh:840: get_is_making_recovery_progress:  test -n ''
       ../qa/workunits/ceph-helpers.sh:988: wait_for_clean:  ((  timer >= 120  ))
       .../qa/workunits/ceph-helpers.sh:993: wait_for_clean:  get_num_active_clean
       .../qa/workunits/ceph-helpers.sh:863: get_num_active_clean:  local 'expression=('
       .../qa/workunits/ceph-helpers.sh:864: get_num_active_clean:  expression+='contains(.,'\''active'\'') and '
       .../qa/workunits/ceph-helpers.sh:865: get_num_active_clean:  expression+='contains(.,'\''clean'\'') and '
       .../qa/workunits/ceph-helpers.sh:866: get_num_active_clean:  expression+='not(contains(.,'\''stale'\''))'
       .../qa/workunits/ceph-helpers.sh:867: get_num_active_clean:  expression+=')'
       .../qa/workunits/ceph-helpers.sh:871: get_num_active_clean:  ceph --format xml pg dump pgs
       .../qa/workunits/ceph-helpers.sh:872: get_num_active_clean:  xmlstarlet sel -t -m '//pg_stat/state[(contains(.,'\''active'\'') and contains(.,'\''clean'\'') and not(contains(.,'\''stale'\'')))]' -v . -n
       .../qa/workunits/ceph-helpers.sh:873: get_num_active_clean:  grep -v '^$'
       .../qa/workunits/ceph-helpers.sh:873: get_num_active_clean:  wc -l
       ../qa/workunits/ceph-helpers.sh:993: wait_for_clean:  cur_active_clean=0
       ../qa/workunits/ceph-helpers.sh:994: wait_for_clean:  test 0 '!=' 0
       ../qa/workunits/ceph-helpers.sh:998: wait_for_clean:  sleep 1
       .../qa/workunits/ceph-helpers.sh:999: wait_for_clean:  expr 67 + 1
       ../qa/workunits/ceph-helpers.sh:999: wait_for_clean:  timer=68
       ../qa/workunits/ceph-helpers.sh:985: wait_for_clean:  is_clean
       .../qa/workunits/ceph-helpers.sh:952: is_clean:  get_num_pgs
       .../qa/workunits/ceph-helpers.sh:898: get_num_pgs:  ceph --format xml status
       .../qa/workunits/ceph-helpers.sh:899: get_num_pgs:  xmlstarlet sel -t -m //pgmap/num_pgs -v .
       ../qa/workunits/ceph-helpers.sh:952: is_clean:  num_pgs=4
       ../qa/workunits/ceph-helpers.sh:953: is_clean:  test 4 '!=' 0
       .../qa/workunits/ceph-helpers.sh:954: is_clean:  get_num_active_clean
       .../qa/workunits/ceph-helpers.sh:863: get_num_active_clean:  local 'expression=('
       .../qa/workunits/ceph-helpers.sh:864: get_num_active_clean:  expression+='contains(.,'\''active'\'') and '
       .../qa/workunits/ceph-helpers.sh:865: get_num_active_clean:  expression+='contains(.,'\''clean'\'') and '
       .../qa/workunits/ceph-helpers.sh:866: get_num_active_clean:  expression+='not(contains(.,'\''stale'\''))'
       .../qa/workunits/ceph-helpers.sh:867: get_num_active_clean:  expression+=')'
       .../qa/workunits/ceph-helpers.sh:871: get_num_active_clean:  ceph --format xml pg dump pgs
       .../qa/workunits/ceph-helpers.sh:872: get_num_active_clean:  xmlstarlet sel -t -m '//pg_stat/state[(contains(.,'\''active'\'') and contains(.,'\''clean'\'') and not(contains(.,'\''stale'\'')))]' -v . -n
       .../qa/workunits/ceph-helpers.sh:873: get_num_active_clean:  grep -v '^$'
       .../qa/workunits/ceph-helpers.sh:873: get_num_active_clean:  wc -l
       ../qa/workunits/ceph-helpers.sh:954: is_clean:  test 0 = 4
       ../qa/workunits/ceph-helpers.sh:954: is_clean:  return 1
       ../qa/workunits/ceph-helpers.sh:986: wait_for_clean:  get_is_making_recovery_progress
       .../qa/workunits/ceph-helpers.sh:839: get_is_making_recovery_progress:  ceph --format xml status
       .../qa/workunits/ceph-helpers.sh:839: get_is_making_recovery_progress:  xmlstarlet sel -t -m //pgmap/recovering_keys_per_sec -v . -o ' ' -t -m //pgmap/recovering_bytes_per_sec -v . -o ' ' -t -m //pgmap/recovering_objects_per_sec -v .
       ../qa/workunits/ceph-helpers.sh:839: get_is_making_recovery_progress:  local progress=
       ../qa/workunits/ceph-helpers.sh:840: get_is_making_recovery_progress:  test -n ''
       ../qa/workunits/ceph-helpers.sh:988: wait_for_clean:  ((  timer >= 120  ))
       .../qa/workunits/ceph-helpers.sh:993: wait_for_clean:  get_num_active_clean
       .../qa/workunits/ceph-helpers.sh:863: get_num_active_clean:  local 'expression=('
       .../qa/workunits/ceph-helpers.sh:864: get_num_active_clean:  expression+='contains(.,'\''active'\'') and '
       .../qa/workunits/ceph-helpers.sh:865: get_num_active_clean:  expression+='contains(.,'\''clean'\'') and '
       .../qa/workunits/ceph-helpers.sh:866: get_num_active_clean:  expression+='not(contains(.,'\''stale'\''))'
       .../qa/workunits/ceph-helpers.sh:867: get_num_active_clean:  expression+=')'
       .../qa/workunits/ceph-helpers.sh:871: get_num_active_clean:  ceph --format xml pg dump pgs
       .../qa/workunits/ceph-helpers.sh:872: get_num_active_clean:  xmlstarlet sel -t -m '//pg_stat/state[(contains(.,'\''active'\'') and contains(.,'\''clean'\'') and not(contains(.,'\''stale'\'')))]' -v . -n
       .../qa/workunits/ceph-helpers.sh:873: get_num_active_clean:  grep -v '^$'
       .../qa/workunits/ceph-helpers.sh:873: get_num_active_clean:  wc -l
       ../qa/workunits/ceph-helpers.sh:993: wait_for_clean:  cur_active_clean=0
       ../qa/workunits/ceph-helpers.sh:994: wait_for_clean:  test 0 '!=' 0
       ../qa/workunits/ceph-helpers.sh:998: wait_for_clean:  sleep 1
       .../qa/workunits/ceph-helpers.sh:999: wait_for_clean:  expr 68 + 1
       ../qa/workunits/ceph-helpers.sh:999: wait_for_clean:  timer=69
       ../qa/workunits/ceph-helpers.sh:985: wait_for_clean:  is_clean
       .../qa/workunits/ceph-helpers.sh:952: is_clean:  get_num_pgs
       .../qa/workunits/ceph-helpers.sh:898: get_num_pgs:  ceph --format xml status
       .../qa/workunits/ceph-helpers.sh:899: get_num_pgs:  xmlstarlet sel -t -m //pgmap/num_pgs -v .
       ../qa/workunits/ceph-helpers.sh:952: is_clean:  num_pgs=4
       ../qa/workunits/ceph-helpers.sh:953: is_clean:  test 4 '!=' 0
       .../qa/workunits/ceph-helpers.sh:954: is_clean:  get_num_active_clean
       .../qa/workunits/ceph-helpers.sh:863: get_num_active_clean:  local 'expression=('
       .../qa/workunits/ceph-helpers.sh:864: get_num_active_clean:  expression+='contains(.,'\''active'\'') and '
       .../qa/workunits/ceph-helpers.sh:865: get_num_active_clean:  expression+='contains(.,'\''clean'\'') and '
       .../qa/workunits/ceph-helpers.sh:866: get_num_active_clean:  expression+='not(contains(.,'\''stale'\''))'
       .../qa/workunits/ceph-helpers.sh:867: get_num_active_clean:  expression+=')'
       .../qa/workunits/ceph-helpers.sh:871: get_num_active_clean:  ceph --format xml pg dump pgs
       .../qa/workunits/ceph-helpers.sh:872: get_num_active_clean:  xmlstarlet sel -t -m '//pg_stat/state[(contains(.,'\''active'\'') and contains(.,'\''clean'\'') and not(contains(.,'\''stale'\'')))]' -v . -n
       .../qa/workunits/ceph-helpers.sh:873: get_num_active_clean:  grep -v '^$'
       .../qa/workunits/ceph-helpers.sh:873: get_num_active_clean:  wc -l
       ../qa/workunits/ceph-helpers.sh:954: is_clean:  test 0 = 4
       ../qa/workunits/ceph-helpers.sh:954: is_clean:  return 1
       ../qa/workunits/ceph-helpers.sh:986: wait_for_clean:  get_is_making_recovery_progress
       .../qa/workunits/ceph-helpers.sh:839: get_is_making_recovery_progress:  ceph --format xml status
       .../qa/workunits/ceph-helpers.sh:839: get_is_making_recovery_progress:  xmlstarlet sel -t -m //pgmap/recovering_keys_per_sec -v . -o ' ' -t -m //pgmap/recovering_bytes_per_sec -v . -o ' ' -t -m //pgmap/recovering_objects_per_sec -v .
       ../qa/workunits/ceph-helpers.sh:839: get_is_making_recovery_progress:  local progress=
       ../qa/workunits/ceph-helpers.sh:840: get_is_making_recovery_progress:  test -n ''
       ../qa/workunits/ceph-helpers.sh:988: wait_for_clean:  ((  timer >= 120  ))
       .../qa/workunits/ceph-helpers.sh:993: wait_for_clean:  get_num_active_clean
       .../qa/workunits/ceph-helpers.sh:863: get_num_active_clean:  local 'expression=('
       .../qa/workunits/ceph-helpers.sh:864: get_num_active_clean:  expression+='contains(.,'\''active'\'') and '
       .../qa/workunits/ceph-helpers.sh:865: get_num_active_clean:  expression+='contains(.,'\''clean'\'') and '
       .../qa/workunits/ceph-helpers.sh:866: get_num_active_clean:  expression+='not(contains(.,'\''stale'\''))'
       .../qa/workunits/ceph-helpers.sh:867: get_num_active_clean:  expression+=')'
       .../qa/workunits/ceph-helpers.sh:871: get_num_active_clean:  ceph --format xml pg dump pgs
       .../qa/workunits/ceph-helpers.sh:872: get_num_active_clean:  xmlstarlet sel -t -m '//pg_stat/state[(contains(.,'\''active'\'') and contains(.,'\''clean'\'') and not(contains(.,'\''stale'\'')))]' -v . -n
       .../qa/workunits/ceph-helpers.sh:873: get_num_active_clean:  grep -v '^$'
       .../qa/workunits/ceph-helpers.sh:873: get_num_active_clean:  wc -l
       ../qa/workunits/ceph-helpers.sh:993: wait_for_clean:  cur_active_clean=0
       ../qa/workunits/ceph-helpers.sh:994: wait_for_clean:  test 0 '!=' 0
       ../qa/workunits/ceph-helpers.sh:998: wait_for_clean:  sleep 1
       .../qa/workunits/ceph-helpers.sh:999: wait_for_clean:  expr 69 + 1
       ../qa/workunits/ceph-helpers.sh:999: wait_for_clean:  timer=70
       ../qa/workunits/ceph-helpers.sh:985: wait_for_clean:  is_clean
       .../qa/workunits/ceph-helpers.sh:952: is_clean:  get_num_pgs
       .../qa/workunits/ceph-helpers.sh:898: get_num_pgs:  ceph --format xml status
       .../qa/workunits/ceph-helpers.sh:899: get_num_pgs:  xmlstarlet sel -t -m //pgmap/num_pgs -v .
       ../qa/workunits/ceph-helpers.sh:952: is_clean:  num_pgs=4
       ../qa/workunits/ceph-helpers.sh:953: is_clean:  test 4 '!=' 0
       .../qa/workunits/ceph-helpers.sh:954: is_clean:  get_num_active_clean
       .../qa/workunits/ceph-helpers.sh:863: get_num_active_clean:  local 'expression=('
       .../qa/workunits/ceph-helpers.sh:864: get_num_active_clean:  expression+='contains(.,'\''active'\'') and '
       .../qa/workunits/ceph-helpers.sh:865: get_num_active_clean:  expression+='contains(.,'\''clean'\'') and '
       .../qa/workunits/ceph-helpers.sh:866: get_num_active_clean:  expression+='not(contains(.,'\''stale'\''))'
       .../qa/workunits/ceph-helpers.sh:867: get_num_active_clean:  expression+=')'
       .../qa/workunits/ceph-helpers.sh:871: get_num_active_clean:  ceph --format xml pg dump pgs
       .../qa/workunits/ceph-helpers.sh:872: get_num_active_clean:  xmlstarlet sel -t -m '//pg_stat/state[(contains(.,'\''active'\'') and contains(.,'\''clean'\'') and not(contains(.,'\''stale'\'')))]' -v . -n
       .../qa/workunits/ceph-helpers.sh:873: get_num_active_clean:  grep -v '^$'
       .../qa/workunits/ceph-helpers.sh:873: get_num_active_clean:  wc -l
       ../qa/workunits/ceph-helpers.sh:954: is_clean:  test 0 = 4
       ../qa/workunits/ceph-helpers.sh:954: is_clean:  return 1
       ../qa/workunits/ceph-helpers.sh:986: wait_for_clean:  get_is_making_recovery_progress
       .../qa/workunits/ceph-helpers.sh:839: get_is_making_recovery_progress:  ceph --format xml status
       .../qa/workunits/ceph-helpers.sh:839: get_is_making_recovery_progress:  xmlstarlet sel -t -m //pgmap/recovering_keys_per_sec -v . -o ' ' -t -m //pgmap/recovering_bytes_per_sec -v . -o ' ' -t -m //pgmap/recovering_objects_per_sec -v .
       ../qa/workunits/ceph-helpers.sh:839: get_is_making_recovery_progress:  local progress=
       ../qa/workunits/ceph-helpers.sh:840: get_is_making_recovery_progress:  test -n ''
       ../qa/workunits/ceph-helpers.sh:988: wait_for_clean:  ((  timer >= 120  ))
       .../qa/workunits/ceph-helpers.sh:993: wait_for_clean:  get_num_active_clean
       .../qa/workunits/ceph-helpers.sh:863: get_num_active_clean:  local 'expression=('
       .../qa/workunits/ceph-helpers.sh:864: get_num_active_clean:  expression+='contains(.,'\''active'\'') and '
       .../qa/workunits/ceph-helpers.sh:865: get_num_active_clean:  expression+='contains(.,'\''clean'\'') and '
       .../qa/workunits/ceph-helpers.sh:866: get_num_active_clean:  expression+='not(contains(.,'\''stale'\''))'
       .../qa/workunits/ceph-helpers.sh:867: get_num_active_clean:  expression+=')'
       .../qa/workunits/ceph-helpers.sh:871: get_num_active_clean:  ceph --format xml pg dump pgs
       .../qa/workunits/ceph-helpers.sh:872: get_num_active_clean:  xmlstarlet sel -t -m '//pg_stat/state[(contains(.,'\''active'\'') and contains(.,'\''clean'\'') and not(contains(.,'\''stale'\'')))]' -v . -n
       .../qa/workunits/ceph-helpers.sh:873: get_num_active_clean:  grep -v '^$'
       .../qa/workunits/ceph-helpers.sh:873: get_num_active_clean:  wc -l
       ../qa/workunits/ceph-helpers.sh:993: wait_for_clean:  cur_active_clean=0
       ../qa/workunits/ceph-helpers.sh:994: wait_for_clean:  test 0 '!=' 0
       ../qa/workunits/ceph-helpers.sh:998: wait_for_clean:  sleep 1
       .../qa/workunits/ceph-helpers.sh:999: wait_for_clean:  expr 70 + 1
       ../qa/workunits/ceph-helpers.sh:999: wait_for_clean:  timer=71
       ../qa/workunits/ceph-helpers.sh:985: wait_for_clean:  is_clean
       .../qa/workunits/ceph-helpers.sh:952: is_clean:  get_num_pgs
       .../qa/workunits/ceph-helpers.sh:898: get_num_pgs:  ceph --format xml status
       .../qa/workunits/ceph-helpers.sh:899: get_num_pgs:  xmlstarlet sel -t -m //pgmap/num_pgs -v .
       ../qa/workunits/ceph-helpers.sh:952: is_clean:  num_pgs=4
       ../qa/workunits/ceph-helpers.sh:953: is_clean:  test 4 '!=' 0
       .../qa/workunits/ceph-helpers.sh:954: is_clean:  get_num_active_clean
       .../qa/workunits/ceph-helpers.sh:863: get_num_active_clean:  local 'expression=('
       .../qa/workunits/ceph-helpers.sh:864: get_num_active_clean:  expression+='contains(.,'\''active'\'') and '
       .../qa/workunits/ceph-helpers.sh:865: get_num_active_clean:  expression+='contains(.,'\''clean'\'') and '
       .../qa/workunits/ceph-helpers.sh:866: get_num_active_clean:  expression+='not(contains(.,'\''stale'\''))'
       .../qa/workunits/ceph-helpers.sh:867: get_num_active_clean:  expression+=')'
       .../qa/workunits/ceph-helpers.sh:871: get_num_active_clean:  ceph --format xml pg dump pgs
       .../qa/workunits/ceph-helpers.sh:872: get_num_active_clean:  xmlstarlet sel -t -m '//pg_stat/state[(contains(.,'\''active'\'') and contains(.,'\''clean'\'') and not(contains(.,'\''stale'\'')))]' -v . -n
       .../qa/workunits/ceph-helpers.sh:873: get_num_active_clean:  grep -v '^$'
       .../qa/workunits/ceph-helpers.sh:873: get_num_active_clean:  wc -l
       ../qa/workunits/ceph-helpers.sh:954: is_clean:  test 0 = 4
       ../qa/workunits/ceph-helpers.sh:954: is_clean:  return 1
       ../qa/workunits/ceph-helpers.sh:986: wait_for_clean:  get_is_making_recovery_progress
       .../qa/workunits/ceph-helpers.sh:839: get_is_making_recovery_progress:  ceph --format xml status
       .../qa/workunits/ceph-helpers.sh:839: get_is_making_recovery_progress:  xmlstarlet sel -t -m //pgmap/recovering_keys_per_sec -v . -o ' ' -t -m //pgmap/recovering_bytes_per_sec -v . -o ' ' -t -m //pgmap/recovering_objects_per_sec -v .
       ../qa/workunits/ceph-helpers.sh:839: get_is_making_recovery_progress:  local progress=
       ../qa/workunits/ceph-helpers.sh:840: get_is_making_recovery_progress:  test -n ''
       ../qa/workunits/ceph-helpers.sh:988: wait_for_clean:  ((  timer >= 120  ))
       .../qa/workunits/ceph-helpers.sh:993: wait_for_clean:  get_num_active_clean
       .../qa/workunits/ceph-helpers.sh:863: get_num_active_clean:  local 'expression=('
       .../qa/workunits/ceph-helpers.sh:864: get_num_active_clean:  expression+='contains(.,'\''active'\'') and '
       .../qa/workunits/ceph-helpers.sh:865: get_num_active_clean:  expression+='contains(.,'\''clean'\'') and '
       .../qa/workunits/ceph-helpers.sh:866: get_num_active_clean:  expression+='not(contains(.,'\''stale'\''))'
       .../qa/workunits/ceph-helpers.sh:867: get_num_active_clean:  expression+=')'
       .../qa/workunits/ceph-helpers.sh:871: get_num_active_clean:  ceph --format xml pg dump pgs
       .../qa/workunits/ceph-helpers.sh:872: get_num_active_clean:  xmlstarlet sel -t -m '//pg_stat/state[(contains(.,'\''active'\'') and contains(.,'\''clean'\'') and not(contains(.,'\''stale'\'')))]' -v . -n
       .../qa/workunits/ceph-helpers.sh:873: get_num_active_clean:  grep -v '^$'
       .../qa/workunits/ceph-helpers.sh:873: get_num_active_clean:  wc -l
       ../qa/workunits/ceph-helpers.sh:993: wait_for_clean:  cur_active_clean=0
       ../qa/workunits/ceph-helpers.sh:994: wait_for_clean:  test 0 '!=' 0
       ../qa/workunits/ceph-helpers.sh:998: wait_for_clean:  sleep 1
       .../qa/workunits/ceph-helpers.sh:999: wait_for_clean:  expr 71 + 1
       ../qa/workunits/ceph-helpers.sh:999: wait_for_clean:  timer=72
       ../qa/workunits/ceph-helpers.sh:985: wait_for_clean:  is_clean
       .../qa/workunits/ceph-helpers.sh:952: is_clean:  get_num_pgs
       .../qa/workunits/ceph-helpers.sh:898: get_num_pgs:  ceph --format xml status
       .../qa/workunits/ceph-helpers.sh:899: get_num_pgs:  xmlstarlet sel -t -m //pgmap/num_pgs -v .
       ../qa/workunits/ceph-helpers.sh:952: is_clean:  num_pgs=4
       ../qa/workunits/ceph-helpers.sh:953: is_clean:  test 4 '!=' 0
       .../qa/workunits/ceph-helpers.sh:954: is_clean:  get_num_active_clean
       .../qa/workunits/ceph-helpers.sh:863: get_num_active_clean:  local 'expression=('
       .../qa/workunits/ceph-helpers.sh:864: get_num_active_clean:  expression+='contains(.,'\''active'\'') and '
       .../qa/workunits/ceph-helpers.sh:865: get_num_active_clean:  expression+='contains(.,'\''clean'\'') and '
       .../qa/workunits/ceph-helpers.sh:866: get_num_active_clean:  expression+='not(contains(.,'\''stale'\''))'
       .../qa/workunits/ceph-helpers.sh:867: get_num_active_clean:  expression+=')'
       .../qa/workunits/ceph-helpers.sh:871: get_num_active_clean:  ceph --format xml pg dump pgs
       .../qa/workunits/ceph-helpers.sh:872: get_num_active_clean:  xmlstarlet sel -t -m '//pg_stat/state[(contains(.,'\''active'\'') and contains(.,'\''clean'\'') and not(contains(.,'\''stale'\'')))]' -v . -n
       .../qa/workunits/ceph-helpers.sh:873: get_num_active_clean:  grep -v '^$'
       .../qa/workunits/ceph-helpers.sh:873: get_num_active_clean:  wc -l
       ../qa/workunits/ceph-helpers.sh:954: is_clean:  test 0 = 4
       ../qa/workunits/ceph-helpers.sh:954: is_clean:  return 1
       ../qa/workunits/ceph-helpers.sh:986: wait_for_clean:  get_is_making_recovery_progress
       .../qa/workunits/ceph-helpers.sh:839: get_is_making_recovery_progress:  ceph --format xml status
       .../qa/workunits/ceph-helpers.sh:839: get_is_making_recovery_progress:  xmlstarlet sel -t -m //pgmap/recovering_keys_per_sec -v . -o ' ' -t -m //pgmap/recovering_bytes_per_sec -v . -o ' ' -t -m //pgmap/recovering_objects_per_sec -v .
       ../qa/workunits/ceph-helpers.sh:839: get_is_making_recovery_progress:  local progress=
       ../qa/workunits/ceph-helpers.sh:840: get_is_making_recovery_progress:  test -n ''
       ../qa/workunits/ceph-helpers.sh:988: wait_for_clean:  ((  timer >= 120  ))
       .../qa/workunits/ceph-helpers.sh:993: wait_for_clean:  get_num_active_clean
       .../qa/workunits/ceph-helpers.sh:863: get_num_active_clean:  local 'expression=('
       .../qa/workunits/ceph-helpers.sh:864: get_num_active_clean:  expression+='contains(.,'\''active'\'') and '
       .../qa/workunits/ceph-helpers.sh:865: get_num_active_clean:  expression+='contains(.,'\''clean'\'') and '
       .../qa/workunits/ceph-helpers.sh:866: get_num_active_clean:  expression+='not(contains(.,'\''stale'\''))'
       .../qa/workunits/ceph-helpers.sh:867: get_num_active_clean:  expression+=')'
       .../qa/workunits/ceph-helpers.sh:871: get_num_active_clean:  ceph --format xml pg dump pgs
       .../qa/workunits/ceph-helpers.sh:873: get_num_active_clean:  grep -v '^$'
       .../qa/workunits/ceph-helpers.sh:872: get_num_active_clean:  xmlstarlet sel -t -m '//pg_stat/state[(contains(.,'\''active'\'') and contains(.,'\''clean'\'') and not(contains(.,'\''stale'\'')))]' -v . -n
       .../qa/workunits/ceph-helpers.sh:873: get_num_active_clean:  wc -l
       ../qa/workunits/ceph-helpers.sh:993: wait_for_clean:  cur_active_clean=0
       ../qa/workunits/ceph-helpers.sh:994: wait_for_clean:  test 0 '!=' 0
       ../qa/workunits/ceph-helpers.sh:998: wait_for_clean:  sleep 1
       .../qa/workunits/ceph-helpers.sh:999: wait_for_clean:  expr 72 + 1
       ../qa/workunits/ceph-helpers.sh:999: wait_for_clean:  timer=73
       ../qa/workunits/ceph-helpers.sh:985: wait_for_clean:  is_clean
       .../qa/workunits/ceph-helpers.sh:952: is_clean:  get_num_pgs
       .../qa/workunits/ceph-helpers.sh:898: get_num_pgs:  ceph --format xml status
       .../qa/workunits/ceph-helpers.sh:899: get_num_pgs:  xmlstarlet sel -t -m //pgmap/num_pgs -v .
       ../qa/workunits/ceph-helpers.sh:952: is_clean:  num_pgs=4
       ../qa/workunits/ceph-helpers.sh:953: is_clean:  test 4 '!=' 0
       .../qa/workunits/ceph-helpers.sh:954: is_clean:  get_num_active_clean
       .../qa/workunits/ceph-helpers.sh:863: get_num_active_clean:  local 'expression=('
       .../qa/workunits/ceph-helpers.sh:864: get_num_active_clean:  expression+='contains(.,'\''active'\'') and '
       .../qa/workunits/ceph-helpers.sh:865: get_num_active_clean:  expression+='contains(.,'\''clean'\'') and '
       .../qa/workunits/ceph-helpers.sh:866: get_num_active_clean:  expression+='not(contains(.,'\''stale'\''))'
       .../qa/workunits/ceph-helpers.sh:867: get_num_active_clean:  expression+=')'
       .../qa/workunits/ceph-helpers.sh:871: get_num_active_clean:  ceph --format xml pg dump pgs
       .../qa/workunits/ceph-helpers.sh:872: get_num_active_clean:  xmlstarlet sel -t -m '//pg_stat/state[(contains(.,'\''active'\'') and contains(.,'\''clean'\'') and not(contains(.,'\''stale'\'')))]' -v . -n
       .../qa/workunits/ceph-helpers.sh:873: get_num_active_clean:  wc -l
       .../qa/workunits/ceph-helpers.sh:873: get_num_active_clean:  grep -v '^$'
       ../qa/workunits/ceph-helpers.sh:954: is_clean:  test 0 = 4
       ../qa/workunits/ceph-helpers.sh:954: is_clean:  return 1
       ../qa/workunits/ceph-helpers.sh:986: wait_for_clean:  get_is_making_recovery_progress
       .../qa/workunits/ceph-helpers.sh:839: get_is_making_recovery_progress:  ceph --format xml status
       .../qa/workunits/ceph-helpers.sh:839: get_is_making_recovery_progress:  xmlstarlet sel -t -m //pgmap/recovering_keys_per_sec -v . -o ' ' -t -m //pgmap/recovering_bytes_per_sec -v . -o ' ' -t -m //pgmap/recovering_objects_per_sec -v .
       ../qa/workunits/ceph-helpers.sh:839: get_is_making_recovery_progress:  local progress=
       ../qa/workunits/ceph-helpers.sh:840: get_is_making_recovery_progress:  test -n ''
       ../qa/workunits/ceph-helpers.sh:988: wait_for_clean:  ((  timer >= 120  ))
       .../qa/workunits/ceph-helpers.sh:993: wait_for_clean:  get_num_active_clean
       .../qa/workunits/ceph-helpers.sh:863: get_num_active_clean:  local 'expression=('
       .../qa/workunits/ceph-helpers.sh:864: get_num_active_clean:  expression+='contains(.,'\''active'\'') and '
       .../qa/workunits/ceph-helpers.sh:865: get_num_active_clean:  expression+='contains(.,'\''clean'\'') and '
       .../qa/workunits/ceph-helpers.sh:866: get_num_active_clean:  expression+='not(contains(.,'\''stale'\''))'
       .../qa/workunits/ceph-helpers.sh:867: get_num_active_clean:  expression+=')'
       .../qa/workunits/ceph-helpers.sh:871: get_num_active_clean:  ceph --format xml pg dump pgs
       .../qa/workunits/ceph-helpers.sh:872: get_num_active_clean:  xmlstarlet sel -t -m '//pg_stat/state[(contains(.,'\''active'\'') and contains(.,'\''clean'\'') and not(contains(.,'\''stale'\'')))]' -v . -n
       .../qa/workunits/ceph-helpers.sh:873: get_num_active_clean:  grep -v '^$'
       .../qa/workunits/ceph-helpers.sh:873: get_num_active_clean:  wc -l
       ../qa/workunits/ceph-helpers.sh:993: wait_for_clean:  cur_active_clean=0
       ../qa/workunits/ceph-helpers.sh:994: wait_for_clean:  test 0 '!=' 0
       ../qa/workunits/ceph-helpers.sh:998: wait_for_clean:  sleep 1
       .../qa/workunits/ceph-helpers.sh:999: wait_for_clean:  expr 73 + 1
       ../qa/workunits/ceph-helpers.sh:999: wait_for_clean:  timer=74
       ../qa/workunits/ceph-helpers.sh:985: wait_for_clean:  is_clean
       .../qa/workunits/ceph-helpers.sh:952: is_clean:  get_num_pgs
       .../qa/workunits/ceph-helpers.sh:898: get_num_pgs:  ceph --format xml status
       .../qa/workunits/ceph-helpers.sh:899: get_num_pgs:  xmlstarlet sel -t -m //pgmap/num_pgs -v .
       ../qa/workunits/ceph-helpers.sh:952: is_clean:  num_pgs=4
       ../qa/workunits/ceph-helpers.sh:953: is_clean:  test 4 '!=' 0
       .../qa/workunits/ceph-helpers.sh:954: is_clean:  get_num_active_clean
       .../qa/workunits/ceph-helpers.sh:863: get_num_active_clean:  local 'expression=('
       .../qa/workunits/ceph-helpers.sh:864: get_num_active_clean:  expression+='contains(.,'\''active'\'') and '
       .../qa/workunits/ceph-helpers.sh:865: get_num_active_clean:  expression+='contains(.,'\''clean'\'') and '
       .../qa/workunits/ceph-helpers.sh:866: get_num_active_clean:  expression+='not(contains(.,'\''stale'\''))'
       .../qa/workunits/ceph-helpers.sh:867: get_num_active_clean:  expression+=')'
       .../qa/workunits/ceph-helpers.sh:871: get_num_active_clean:  ceph --format xml pg dump pgs
       .../qa/workunits/ceph-helpers.sh:872: get_num_active_clean:  xmlstarlet sel -t -m '//pg_stat/state[(contains(.,'\''active'\'') and contains(.,'\''clean'\'') and not(contains(.,'\''stale'\'')))]' -v . -n
       .../qa/workunits/ceph-helpers.sh:873: get_num_active_clean:  grep -v '^$'
       .../qa/workunits/ceph-helpers.sh:873: get_num_active_clean:  wc -l
       ../qa/workunits/ceph-helpers.sh:954: is_clean:  test 0 = 4
       ../qa/workunits/ceph-helpers.sh:954: is_clean:  return 1
       ../qa/workunits/ceph-helpers.sh:986: wait_for_clean:  get_is_making_recovery_progress
       .../qa/workunits/ceph-helpers.sh:839: get_is_making_recovery_progress:  ceph --format xml status
       .../qa/workunits/ceph-helpers.sh:839: get_is_making_recovery_progress:  xmlstarlet sel -t -m //pgmap/recovering_keys_per_sec -v . -o ' ' -t -m //pgmap/recovering_bytes_per_sec -v . -o ' ' -t -m //pgmap/recovering_objects_per_sec -v .
       ../qa/workunits/ceph-helpers.sh:839: get_is_making_recovery_progress:  local progress=
       ../qa/workunits/ceph-helpers.sh:840: get_is_making_recovery_progress:  test -n ''
       ../qa/workunits/ceph-helpers.sh:988: wait_for_clean:  ((  timer >= 120  ))
       .../qa/workunits/ceph-helpers.sh:993: wait_for_clean:  get_num_active_clean
       .../qa/workunits/ceph-helpers.sh:863: get_num_active_clean:  local 'expression=('
       .../qa/workunits/ceph-helpers.sh:864: get_num_active_clean:  expression+='contains(.,'\''active'\'') and '
       .../qa/workunits/ceph-helpers.sh:865: get_num_active_clean:  expression+='contains(.,'\''clean'\'') and '
       .../qa/workunits/ceph-helpers.sh:866: get_num_active_clean:  expression+='not(contains(.,'\''stale'\''))'
       .../qa/workunits/ceph-helpers.sh:867: get_num_active_clean:  expression+=')'
       .../qa/workunits/ceph-helpers.sh:871: get_num_active_clean:  ceph --format xml pg dump pgs
       .../qa/workunits/ceph-helpers.sh:872: get_num_active_clean:  xmlstarlet sel -t -m '//pg_stat/state[(contains(.,'\''active'\'') and contains(.,'\''clean'\'') and not(contains(.,'\''stale'\'')))]' -v . -n
       .../qa/workunits/ceph-helpers.sh:873: get_num_active_clean:  grep -v '^$'
       .../qa/workunits/ceph-helpers.sh:873: get_num_active_clean:  wc -l
       ../qa/workunits/ceph-helpers.sh:993: wait_for_clean:  cur_active_clean=0
       ../qa/workunits/ceph-helpers.sh:994: wait_for_clean:  test 0 '!=' 0
       ../qa/workunits/ceph-helpers.sh:998: wait_for_clean:  sleep 1
       .../qa/workunits/ceph-helpers.sh:999: wait_for_clean:  expr 74 + 1
       ../qa/workunits/ceph-helpers.sh:999: wait_for_clean:  timer=75
       ../qa/workunits/ceph-helpers.sh:985: wait_for_clean:  is_clean
       .../qa/workunits/ceph-helpers.sh:952: is_clean:  get_num_pgs
       .../qa/workunits/ceph-helpers.sh:898: get_num_pgs:  ceph --format xml status
       .../qa/workunits/ceph-helpers.sh:899: get_num_pgs:  xmlstarlet sel -t -m //pgmap/num_pgs -v .
       ../qa/workunits/ceph-helpers.sh:952: is_clean:  num_pgs=4
       ../qa/workunits/ceph-helpers.sh:953: is_clean:  test 4 '!=' 0
       .../qa/workunits/ceph-helpers.sh:954: is_clean:  get_num_active_clean
       .../qa/workunits/ceph-helpers.sh:863: get_num_active_clean:  local 'expression=('
       .../qa/workunits/ceph-helpers.sh:864: get_num_active_clean:  expression+='contains(.,'\''active'\'') and '
       .../qa/workunits/ceph-helpers.sh:865: get_num_active_clean:  expression+='contains(.,'\''clean'\'') and '
       .../qa/workunits/ceph-helpers.sh:866: get_num_active_clean:  expression+='not(contains(.,'\''stale'\''))'
       .../qa/workunits/ceph-helpers.sh:867: get_num_active_clean:  expression+=')'
       .../qa/workunits/ceph-helpers.sh:871: get_num_active_clean:  ceph --format xml pg dump pgs
       .../qa/workunits/ceph-helpers.sh:872: get_num_active_clean:  xmlstarlet sel -t -m '//pg_stat/state[(contains(.,'\''active'\'') and contains(.,'\''clean'\'') and not(contains(.,'\''stale'\'')))]' -v . -n
       .../qa/workunits/ceph-helpers.sh:873: get_num_active_clean:  wc -l
       .../qa/workunits/ceph-helpers.sh:873: get_num_active_clean:  grep -v '^$'
       ../qa/workunits/ceph-helpers.sh:954: is_clean:  test 0 = 4
       ../qa/workunits/ceph-helpers.sh:954: is_clean:  return 1
       ../qa/workunits/ceph-helpers.sh:986: wait_for_clean:  get_is_making_recovery_progress
       .../qa/workunits/ceph-helpers.sh:839: get_is_making_recovery_progress:  ceph --format xml status
       .../qa/workunits/ceph-helpers.sh:839: get_is_making_recovery_progress:  xmlstarlet sel -t -m //pgmap/recovering_keys_per_sec -v . -o ' ' -t -m //pgmap/recovering_bytes_per_sec -v . -o ' ' -t -m //pgmap/recovering_objects_per_sec -v .
       ../qa/workunits/ceph-helpers.sh:839: get_is_making_recovery_progress:  local progress=
       ../qa/workunits/ceph-helpers.sh:840: get_is_making_recovery_progress:  test -n ''
       ../qa/workunits/ceph-helpers.sh:988: wait_for_clean:  ((  timer >= 120  ))
       .../qa/workunits/ceph-helpers.sh:993: wait_for_clean:  get_num_active_clean
       .../qa/workunits/ceph-helpers.sh:863: get_num_active_clean:  local 'expression=('
       .../qa/workunits/ceph-helpers.sh:864: get_num_active_clean:  expression+='contains(.,'\''active'\'') and '
       .../qa/workunits/ceph-helpers.sh:865: get_num_active_clean:  expression+='contains(.,'\''clean'\'') and '
       .../qa/workunits/ceph-helpers.sh:866: get_num_active_clean:  expression+='not(contains(.,'\''stale'\''))'
       .../qa/workunits/ceph-helpers.sh:867: get_num_active_clean:  expression+=')'
       .../qa/workunits/ceph-helpers.sh:871: get_num_active_clean:  ceph --format xml pg dump pgs
       .../qa/workunits/ceph-helpers.sh:872: get_num_active_clean:  xmlstarlet sel -t -m '//pg_stat/state[(contains(.,'\''active'\'') and contains(.,'\''clean'\'') and not(contains(.,'\''stale'\'')))]' -v . -n
       .../qa/workunits/ceph-helpers.sh:873: get_num_active_clean:  wc -l
       .../qa/workunits/ceph-helpers.sh:873: get_num_active_clean:  grep -v '^$'
       ../qa/workunits/ceph-helpers.sh:993: wait_for_clean:  cur_active_clean=0
       ../qa/workunits/ceph-helpers.sh:994: wait_for_clean:  test 0 '!=' 0
       ../qa/workunits/ceph-helpers.sh:998: wait_for_clean:  sleep 1
       .../qa/workunits/ceph-helpers.sh:999: wait_for_clean:  expr 75 + 1
       ../qa/workunits/ceph-helpers.sh:999: wait_for_clean:  timer=76
       ../qa/workunits/ceph-helpers.sh:985: wait_for_clean:  is_clean
       .../qa/workunits/ceph-helpers.sh:952: is_clean:  get_num_pgs
       .../qa/workunits/ceph-helpers.sh:898: get_num_pgs:  ceph --format xml status
       .../qa/workunits/ceph-helpers.sh:899: get_num_pgs:  xmlstarlet sel -t -m //pgmap/num_pgs -v .
       ../qa/workunits/ceph-helpers.sh:952: is_clean:  num_pgs=4
       ../qa/workunits/ceph-helpers.sh:953: is_clean:  test 4 '!=' 0
       .../qa/workunits/ceph-helpers.sh:954: is_clean:  get_num_active_clean
       .../qa/workunits/ceph-helpers.sh:863: get_num_active_clean:  local 'expression=('
       .../qa/workunits/ceph-helpers.sh:864: get_num_active_clean:  expression+='contains(.,'\''active'\'') and '
       .../qa/workunits/ceph-helpers.sh:865: get_num_active_clean:  expression+='contains(.,'\''clean'\'') and '
       .../qa/workunits/ceph-helpers.sh:866: get_num_active_clean:  expression+='not(contains(.,'\''stale'\''))'
       .../qa/workunits/ceph-helpers.sh:867: get_num_active_clean:  expression+=')'
       .../qa/workunits/ceph-helpers.sh:871: get_num_active_clean:  ceph --format xml pg dump pgs
       .../qa/workunits/ceph-helpers.sh:872: get_num_active_clean:  xmlstarlet sel -t -m '//pg_stat/state[(contains(.,'\''active'\'') and contains(.,'\''clean'\'') and not(contains(.,'\''stale'\'')))]' -v . -n
       .../qa/workunits/ceph-helpers.sh:873: get_num_active_clean:  grep -v '^$'
       .../qa/workunits/ceph-helpers.sh:873: get_num_active_clean:  wc -l
       ../qa/workunits/ceph-helpers.sh:954: is_clean:  test 0 = 4
       ../qa/workunits/ceph-helpers.sh:954: is_clean:  return 1
       ../qa/workunits/ceph-helpers.sh:986: wait_for_clean:  get_is_making_recovery_progress
       .../qa/workunits/ceph-helpers.sh:839: get_is_making_recovery_progress:  ceph --format xml status
       .../qa/workunits/ceph-helpers.sh:839: get_is_making_recovery_progress:  xmlstarlet sel -t -m //pgmap/recovering_keys_per_sec -v . -o ' ' -t -m //pgmap/recovering_bytes_per_sec -v . -o ' ' -t -m //pgmap/recovering_objects_per_sec -v .
       ../qa/workunits/ceph-helpers.sh:839: get_is_making_recovery_progress:  local progress=
       ../qa/workunits/ceph-helpers.sh:840: get_is_making_recovery_progress:  test -n ''
       ../qa/workunits/ceph-helpers.sh:988: wait_for_clean:  ((  timer >= 120  ))
       .../qa/workunits/ceph-helpers.sh:993: wait_for_clean:  get_num_active_clean
       .../qa/workunits/ceph-helpers.sh:863: get_num_active_clean:  local 'expression=('
       .../qa/workunits/ceph-helpers.sh:864: get_num_active_clean:  expression+='contains(.,'\''active'\'') and '
       .../qa/workunits/ceph-helpers.sh:865: get_num_active_clean:  expression+='contains(.,'\''clean'\'') and '
       .../qa/workunits/ceph-helpers.sh:866: get_num_active_clean:  expression+='not(contains(.,'\''stale'\''))'
       .../qa/workunits/ceph-helpers.sh:867: get_num_active_clean:  expression+=')'
       .../qa/workunits/ceph-helpers.sh:871: get_num_active_clean:  ceph --format xml pg dump pgs
       .../qa/workunits/ceph-helpers.sh:872: get_num_active_clean:  xmlstarlet sel -t -m '//pg_stat/state[(contains(.,'\''active'\'') and contains(.,'\''clean'\'') and not(contains(.,'\''stale'\'')))]' -v . -n
       .../qa/workunits/ceph-helpers.sh:873: get_num_active_clean:  grep -v '^$'
       .../qa/workunits/ceph-helpers.sh:873: get_num_active_clean:  wc -l
       ../qa/workunits/ceph-helpers.sh:993: wait_for_clean:  cur_active_clean=0
       ../qa/workunits/ceph-helpers.sh:994: wait_for_clean:  test 0 '!=' 0
       ../qa/workunits/ceph-helpers.sh:998: wait_for_clean:  sleep 1
       .../qa/workunits/ceph-helpers.sh:999: wait_for_clean:  expr 76 + 1
       ../qa/workunits/ceph-helpers.sh:999: wait_for_clean:  timer=77
       ../qa/workunits/ceph-helpers.sh:985: wait_for_clean:  is_clean
       .../qa/workunits/ceph-helpers.sh:952: is_clean:  get_num_pgs
       .../qa/workunits/ceph-helpers.sh:898: get_num_pgs:  ceph --format xml status
       .../qa/workunits/ceph-helpers.sh:899: get_num_pgs:  xmlstarlet sel -t -m //pgmap/num_pgs -v .
       ../qa/workunits/ceph-helpers.sh:952: is_clean:  num_pgs=4
       ../qa/workunits/ceph-helpers.sh:953: is_clean:  test 4 '!=' 0
       .../qa/workunits/ceph-helpers.sh:954: is_clean:  get_num_active_clean
       .../qa/workunits/ceph-helpers.sh:863: get_num_active_clean:  local 'expression=('
       .../qa/workunits/ceph-helpers.sh:864: get_num_active_clean:  expression+='contains(.,'\''active'\'') and '
       .../qa/workunits/ceph-helpers.sh:865: get_num_active_clean:  expression+='contains(.,'\''clean'\'') and '
       .../qa/workunits/ceph-helpers.sh:866: get_num_active_clean:  expression+='not(contains(.,'\''stale'\''))'
       .../qa/workunits/ceph-helpers.sh:867: get_num_active_clean:  expression+=')'
       .../qa/workunits/ceph-helpers.sh:871: get_num_active_clean:  ceph --format xml pg dump pgs
       .../qa/workunits/ceph-helpers.sh:872: get_num_active_clean:  xmlstarlet sel -t -m '//pg_stat/state[(contains(.,'\''active'\'') and contains(.,'\''clean'\'') and not(contains(.,'\''stale'\'')))]' -v . -n
       .../qa/workunits/ceph-helpers.sh:873: get_num_active_clean:  grep -v '^$'
       .../qa/workunits/ceph-helpers.sh:873: get_num_active_clean:  wc -l
       ../qa/workunits/ceph-helpers.sh:954: is_clean:  test 0 = 4
       ../qa/workunits/ceph-helpers.sh:954: is_clean:  return 1
       ../qa/workunits/ceph-helpers.sh:986: wait_for_clean:  get_is_making_recovery_progress
       .../qa/workunits/ceph-helpers.sh:839: get_is_making_recovery_progress:  ceph --format xml status
       .../qa/workunits/ceph-helpers.sh:839: get_is_making_recovery_progress:  xmlstarlet sel -t -m //pgmap/recovering_keys_per_sec -v . -o ' ' -t -m //pgmap/recovering_bytes_per_sec -v . -o ' ' -t -m //pgmap/recovering_objects_per_sec -v .
       ../qa/workunits/ceph-helpers.sh:839: get_is_making_recovery_progress:  local progress=
       ../qa/workunits/ceph-helpers.sh:840: get_is_making_recovery_progress:  test -n ''
       ../qa/workunits/ceph-helpers.sh:988: wait_for_clean:  ((  timer >= 120  ))
       .../qa/workunits/ceph-helpers.sh:993: wait_for_clean:  get_num_active_clean
       .../qa/workunits/ceph-helpers.sh:863: get_num_active_clean:  local 'expression=('
       .../qa/workunits/ceph-helpers.sh:864: get_num_active_clean:  expression+='contains(.,'\''active'\'') and '
       .../qa/workunits/ceph-helpers.sh:865: get_num_active_clean:  expression+='contains(.,'\''clean'\'') and '
       .../qa/workunits/ceph-helpers.sh:866: get_num_active_clean:  expression+='not(contains(.,'\''stale'\''))'
       .../qa/workunits/ceph-helpers.sh:867: get_num_active_clean:  expression+=')'
       .../qa/workunits/ceph-helpers.sh:871: get_num_active_clean:  ceph --format xml pg dump pgs
       .../qa/workunits/ceph-helpers.sh:872: get_num_active_clean:  xmlstarlet sel -t -m '//pg_stat/state[(contains(.,'\''active'\'') and contains(.,'\''clean'\'') and not(contains(.,'\''stale'\'')))]' -v . -n
       .../qa/workunits/ceph-helpers.sh:873: get_num_active_clean:  grep -v '^$'
       .../qa/workunits/ceph-helpers.sh:873: get_num_active_clean:  wc -l
       ../qa/workunits/ceph-helpers.sh:993: wait_for_clean:  cur_active_clean=0
       ../qa/workunits/ceph-helpers.sh:994: wait_for_clean:  test 0 '!=' 0
       ../qa/workunits/ceph-helpers.sh:998: wait_for_clean:  sleep 1
       .../qa/workunits/ceph-helpers.sh:999: wait_for_clean:  expr 77 + 1
       ../qa/workunits/ceph-helpers.sh:999: wait_for_clean:  timer=78
       ../qa/workunits/ceph-helpers.sh:985: wait_for_clean:  is_clean
       .../qa/workunits/ceph-helpers.sh:952: is_clean:  get_num_pgs
       .../qa/workunits/ceph-helpers.sh:898: get_num_pgs:  ceph --format xml status
       .../qa/workunits/ceph-helpers.sh:899: get_num_pgs:  xmlstarlet sel -t -m //pgmap/num_pgs -v .
       ../qa/workunits/ceph-helpers.sh:952: is_clean:  num_pgs=4
       ../qa/workunits/ceph-helpers.sh:953: is_clean:  test 4 '!=' 0
       .../qa/workunits/ceph-helpers.sh:954: is_clean:  get_num_active_clean
       .../qa/workunits/ceph-helpers.sh:863: get_num_active_clean:  local 'expression=('
       .../qa/workunits/ceph-helpers.sh:864: get_num_active_clean:  expression+='contains(.,'\''active'\'') and '
       .../qa/workunits/ceph-helpers.sh:865: get_num_active_clean:  expression+='contains(.,'\''clean'\'') and '
       .../qa/workunits/ceph-helpers.sh:866: get_num_active_clean:  expression+='not(contains(.,'\''stale'\''))'
       .../qa/workunits/ceph-helpers.sh:867: get_num_active_clean:  expression+=')'
       .../qa/workunits/ceph-helpers.sh:871: get_num_active_clean:  ceph --format xml pg dump pgs
       .../qa/workunits/ceph-helpers.sh:872: get_num_active_clean:  xmlstarlet sel -t -m '//pg_stat/state[(contains(.,'\''active'\'') and contains(.,'\''clean'\'') and not(contains(.,'\''stale'\'')))]' -v . -n
       .../qa/workunits/ceph-helpers.sh:873: get_num_active_clean:  grep -v '^$'
       .../qa/workunits/ceph-helpers.sh:873: get_num_active_clean:  wc -l
       ../qa/workunits/ceph-helpers.sh:954: is_clean:  test 0 = 4
       ../qa/workunits/ceph-helpers.sh:954: is_clean:  return 1
       ../qa/workunits/ceph-helpers.sh:986: wait_for_clean:  get_is_making_recovery_progress
       .../qa/workunits/ceph-helpers.sh:839: get_is_making_recovery_progress:  ceph --format xml status
       .../qa/workunits/ceph-helpers.sh:839: get_is_making_recovery_progress:  xmlstarlet sel -t -m //pgmap/recovering_keys_per_sec -v . -o ' ' -t -m //pgmap/recovering_bytes_per_sec -v . -o ' ' -t -m //pgmap/recovering_objects_per_sec -v .
       ../qa/workunits/ceph-helpers.sh:839: get_is_making_recovery_progress:  local progress=
       ../qa/workunits/ceph-helpers.sh:840: get_is_making_recovery_progress:  test -n ''
       ../qa/workunits/ceph-helpers.sh:988: wait_for_clean:  ((  timer >= 120  ))
       .../qa/workunits/ceph-helpers.sh:993: wait_for_clean:  get_num_active_clean
       .../qa/workunits/ceph-helpers.sh:863: get_num_active_clean:  local 'expression=('
       .../qa/workunits/ceph-helpers.sh:864: get_num_active_clean:  expression+='contains(.,'\''active'\'') and '
       .../qa/workunits/ceph-helpers.sh:865: get_num_active_clean:  expression+='contains(.,'\''clean'\'') and '
       .../qa/workunits/ceph-helpers.sh:866: get_num_active_clean:  expression+='not(contains(.,'\''stale'\''))'
       .../qa/workunits/ceph-helpers.sh:867: get_num_active_clean:  expression+=')'
       .../qa/workunits/ceph-helpers.sh:871: get_num_active_clean:  ceph --format xml pg dump pgs
       .../qa/workunits/ceph-helpers.sh:872: get_num_active_clean:  xmlstarlet sel -t -m '//pg_stat/state[(contains(.,'\''active'\'') and contains(.,'\''clean'\'') and not(contains(.,'\''stale'\'')))]' -v . -n
       .../qa/workunits/ceph-helpers.sh:873: get_num_active_clean:  grep -v '^$'
       .../qa/workunits/ceph-helpers.sh:873: get_num_active_clean:  wc -l
       ../qa/workunits/ceph-helpers.sh:993: wait_for_clean:  cur_active_clean=0
       ../qa/workunits/ceph-helpers.sh:994: wait_for_clean:  test 0 '!=' 0
       ../qa/workunits/ceph-helpers.sh:998: wait_for_clean:  sleep 1
       .../qa/workunits/ceph-helpers.sh:999: wait_for_clean:  expr 78 + 1
       ../qa/workunits/ceph-helpers.sh:999: wait_for_clean:  timer=79
       ../qa/workunits/ceph-helpers.sh:985: wait_for_clean:  is_clean
       .../qa/workunits/ceph-helpers.sh:952: is_clean:  get_num_pgs
       .../qa/workunits/ceph-helpers.sh:898: get_num_pgs:  ceph --format xml status
       .../qa/workunits/ceph-helpers.sh:899: get_num_pgs:  xmlstarlet sel -t -m //pgmap/num_pgs -v .
       ../qa/workunits/ceph-helpers.sh:952: is_clean:  num_pgs=4
       ../qa/workunits/ceph-helpers.sh:953: is_clean:  test 4 '!=' 0
       .../qa/workunits/ceph-helpers.sh:954: is_clean:  get_num_active_clean
       .../qa/workunits/ceph-helpers.sh:863: get_num_active_clean:  local 'expression=('
       .../qa/workunits/ceph-helpers.sh:864: get_num_active_clean:  expression+='contains(.,'\''active'\'') and '
       .../qa/workunits/ceph-helpers.sh:865: get_num_active_clean:  expression+='contains(.,'\''clean'\'') and '
       .../qa/workunits/ceph-helpers.sh:866: get_num_active_clean:  expression+='not(contains(.,'\''stale'\''))'
       .../qa/workunits/ceph-helpers.sh:867: get_num_active_clean:  expression+=')'
       .../qa/workunits/ceph-helpers.sh:871: get_num_active_clean:  ceph --format xml pg dump pgs
       .../qa/workunits/ceph-helpers.sh:872: get_num_active_clean:  xmlstarlet sel -t -m '//pg_stat/state[(contains(.,'\''active'\'') and contains(.,'\''clean'\'') and not(contains(.,'\''stale'\'')))]' -v . -n
       .../qa/workunits/ceph-helpers.sh:873: get_num_active_clean:  grep -v '^$'
       .../qa/workunits/ceph-helpers.sh:873: get_num_active_clean:  wc -l
       ../qa/workunits/ceph-helpers.sh:954: is_clean:  test 0 = 4
       ../qa/workunits/ceph-helpers.sh:954: is_clean:  return 1
       ../qa/workunits/ceph-helpers.sh:986: wait_for_clean:  get_is_making_recovery_progress
       .../qa/workunits/ceph-helpers.sh:839: get_is_making_recovery_progress:  ceph --format xml status
       .../qa/workunits/ceph-helpers.sh:839: get_is_making_recovery_progress:  xmlstarlet sel -t -m //pgmap/recovering_keys_per_sec -v . -o ' ' -t -m //pgmap/recovering_bytes_per_sec -v . -o ' ' -t -m //pgmap/recovering_objects_per_sec -v .
       ../qa/workunits/ceph-helpers.sh:839: get_is_making_recovery_progress:  local progress=
       ../qa/workunits/ceph-helpers.sh:840: get_is_making_recovery_progress:  test -n ''
       ../qa/workunits/ceph-helpers.sh:988: wait_for_clean:  ((  timer >= 120  ))
       .../qa/workunits/ceph-helpers.sh:993: wait_for_clean:  get_num_active_clean
       .../qa/workunits/ceph-helpers.sh:863: get_num_active_clean:  local 'expression=('
       .../qa/workunits/ceph-helpers.sh:864: get_num_active_clean:  expression+='contains(.,'\''active'\'') and '
       .../qa/workunits/ceph-helpers.sh:865: get_num_active_clean:  expression+='contains(.,'\''clean'\'') and '
       .../qa/workunits/ceph-helpers.sh:866: get_num_active_clean:  expression+='not(contains(.,'\''stale'\''))'
       .../qa/workunits/ceph-helpers.sh:867: get_num_active_clean:  expression+=')'
       .../qa/workunits/ceph-helpers.sh:871: get_num_active_clean:  ceph --format xml pg dump pgs
       .../qa/workunits/ceph-helpers.sh:873: get_num_active_clean:  grep -v '^$'
       .../qa/workunits/ceph-helpers.sh:872: get_num_active_clean:  xmlstarlet sel -t -m '//pg_stat/state[(contains(.,'\''active'\'') and contains(.,'\''clean'\'') and not(contains(.,'\''stale'\'')))]' -v . -n
       .../qa/workunits/ceph-helpers.sh:873: get_num_active_clean:  wc -l
       ../qa/workunits/ceph-helpers.sh:993: wait_for_clean:  cur_active_clean=0
       ../qa/workunits/ceph-helpers.sh:994: wait_for_clean:  test 0 '!=' 0
       ../qa/workunits/ceph-helpers.sh:998: wait_for_clean:  sleep 1
       .../qa/workunits/ceph-helpers.sh:999: wait_for_clean:  expr 79 + 1
       ../qa/workunits/ceph-helpers.sh:999: wait_for_clean:  timer=80
       ../qa/workunits/ceph-helpers.sh:985: wait_for_clean:  is_clean
       .../qa/workunits/ceph-helpers.sh:952: is_clean:  get_num_pgs
       .../qa/workunits/ceph-helpers.sh:898: get_num_pgs:  ceph --format xml status
       .../qa/workunits/ceph-helpers.sh:899: get_num_pgs:  xmlstarlet sel -t -m //pgmap/num_pgs -v .
       ../qa/workunits/ceph-helpers.sh:952: is_clean:  num_pgs=4
       ../qa/workunits/ceph-helpers.sh:953: is_clean:  test 4 '!=' 0
       .../qa/workunits/ceph-helpers.sh:954: is_clean:  get_num_active_clean
       .../qa/workunits/ceph-helpers.sh:863: get_num_active_clean:  local 'expression=('
       .../qa/workunits/ceph-helpers.sh:864: get_num_active_clean:  expression+='contains(.,'\''active'\'') and '
       .../qa/workunits/ceph-helpers.sh:865: get_num_active_clean:  expression+='contains(.,'\''clean'\'') and '
       .../qa/workunits/ceph-helpers.sh:866: get_num_active_clean:  expression+='not(contains(.,'\''stale'\''))'
       .../qa/workunits/ceph-helpers.sh:867: get_num_active_clean:  expression+=')'
       .../qa/workunits/ceph-helpers.sh:871: get_num_active_clean:  ceph --format xml pg dump pgs
       .../qa/workunits/ceph-helpers.sh:872: get_num_active_clean:  xmlstarlet sel -t -m '//pg_stat/state[(contains(.,'\''active'\'') and contains(.,'\''clean'\'') and not(contains(.,'\''stale'\'')))]' -v . -n
       .../qa/workunits/ceph-helpers.sh:873: get_num_active_clean:  wc -l
       .../qa/workunits/ceph-helpers.sh:873: get_num_active_clean:  grep -v '^$'
       ../qa/workunits/ceph-helpers.sh:954: is_clean:  test 0 = 4
       ../qa/workunits/ceph-helpers.sh:954: is_clean:  return 1
       ../qa/workunits/ceph-helpers.sh:986: wait_for_clean:  get_is_making_recovery_progress
       .../qa/workunits/ceph-helpers.sh:839: get_is_making_recovery_progress:  ceph --format xml status
       .../qa/workunits/ceph-helpers.sh:839: get_is_making_recovery_progress:  xmlstarlet sel -t -m //pgmap/recovering_keys_per_sec -v . -o ' ' -t -m //pgmap/recovering_bytes_per_sec -v . -o ' ' -t -m //pgmap/recovering_objects_per_sec -v .
       ../qa/workunits/ceph-helpers.sh:839: get_is_making_recovery_progress:  local progress=
       ../qa/workunits/ceph-helpers.sh:840: get_is_making_recovery_progress:  test -n ''
       ../qa/workunits/ceph-helpers.sh:988: wait_for_clean:  ((  timer >= 120  ))
       .../qa/workunits/ceph-helpers.sh:993: wait_for_clean:  get_num_active_clean
       .../qa/workunits/ceph-helpers.sh:863: get_num_active_clean:  local 'expression=('
       .../qa/workunits/ceph-helpers.sh:864: get_num_active_clean:  expression+='contains(.,'\''active'\'') and '
       .../qa/workunits/ceph-helpers.sh:865: get_num_active_clean:  expression+='contains(.,'\''clean'\'') and '
       .../qa/workunits/ceph-helpers.sh:866: get_num_active_clean:  expression+='not(contains(.,'\''stale'\''))'
       .../qa/workunits/ceph-helpers.sh:867: get_num_active_clean:  expression+=')'
       .../qa/workunits/ceph-helpers.sh:871: get_num_active_clean:  ceph --format xml pg dump pgs
       .../qa/workunits/ceph-helpers.sh:872: get_num_active_clean:  xmlstarlet sel -t -m '//pg_stat/state[(contains(.,'\''active'\'') and contains(.,'\''clean'\'') and not(contains(.,'\''stale'\'')))]' -v . -n
       .../qa/workunits/ceph-helpers.sh:873: get_num_active_clean:  grep -v '^$'
       .../qa/workunits/ceph-helpers.sh:873: get_num_active_clean:  wc -l
       ../qa/workunits/ceph-helpers.sh:993: wait_for_clean:  cur_active_clean=0
       ../qa/workunits/ceph-helpers.sh:994: wait_for_clean:  test 0 '!=' 0
       ../qa/workunits/ceph-helpers.sh:998: wait_for_clean:  sleep 1
       .../qa/workunits/ceph-helpers.sh:999: wait_for_clean:  expr 80 + 1
       ../qa/workunits/ceph-helpers.sh:999: wait_for_clean:  timer=81
       ../qa/workunits/ceph-helpers.sh:985: wait_for_clean:  is_clean
       .../qa/workunits/ceph-helpers.sh:952: is_clean:  get_num_pgs
       .../qa/workunits/ceph-helpers.sh:898: get_num_pgs:  ceph --format xml status
       .../qa/workunits/ceph-helpers.sh:899: get_num_pgs:  xmlstarlet sel -t -m //pgmap/num_pgs -v .
       ../qa/workunits/ceph-helpers.sh:952: is_clean:  num_pgs=4
       ../qa/workunits/ceph-helpers.sh:953: is_clean:  test 4 '!=' 0
       .../qa/workunits/ceph-helpers.sh:954: is_clean:  get_num_active_clean
       .../qa/workunits/ceph-helpers.sh:863: get_num_active_clean:  local 'expression=('
       .../qa/workunits/ceph-helpers.sh:864: get_num_active_clean:  expression+='contains(.,'\''active'\'') and '
       .../qa/workunits/ceph-helpers.sh:865: get_num_active_clean:  expression+='contains(.,'\''clean'\'') and '
       .../qa/workunits/ceph-helpers.sh:866: get_num_active_clean:  expression+='not(contains(.,'\''stale'\''))'
       .../qa/workunits/ceph-helpers.sh:867: get_num_active_clean:  expression+=')'
       .../qa/workunits/ceph-helpers.sh:871: get_num_active_clean:  ceph --format xml pg dump pgs
       .../qa/workunits/ceph-helpers.sh:872: get_num_active_clean:  xmlstarlet sel -t -m '//pg_stat/state[(contains(.,'\''active'\'') and contains(.,'\''clean'\'') and not(contains(.,'\''stale'\'')))]' -v . -n
       .../qa/workunits/ceph-helpers.sh:873: get_num_active_clean:  grep -v '^$'
       .../qa/workunits/ceph-helpers.sh:873: get_num_active_clean:  wc -l
       ../qa/workunits/ceph-helpers.sh:954: is_clean:  test 0 = 4
       ../qa/workunits/ceph-helpers.sh:954: is_clean:  return 1
       ../qa/workunits/ceph-helpers.sh:986: wait_for_clean:  get_is_making_recovery_progress
       .../qa/workunits/ceph-helpers.sh:839: get_is_making_recovery_progress:  ceph --format xml status
       .../qa/workunits/ceph-helpers.sh:839: get_is_making_recovery_progress:  xmlstarlet sel -t -m //pgmap/recovering_keys_per_sec -v . -o ' ' -t -m //pgmap/recovering_bytes_per_sec -v . -o ' ' -t -m //pgmap/recovering_objects_per_sec -v .
       ../qa/workunits/ceph-helpers.sh:839: get_is_making_recovery_progress:  local progress=
       ../qa/workunits/ceph-helpers.sh:840: get_is_making_recovery_progress:  test -n ''
       ../qa/workunits/ceph-helpers.sh:988: wait_for_clean:  ((  timer >= 120  ))
       .../qa/workunits/ceph-helpers.sh:993: wait_for_clean:  get_num_active_clean
       .../qa/workunits/ceph-helpers.sh:863: get_num_active_clean:  local 'expression=('
       .../qa/workunits/ceph-helpers.sh:864: get_num_active_clean:  expression+='contains(.,'\''active'\'') and '
       .../qa/workunits/ceph-helpers.sh:865: get_num_active_clean:  expression+='contains(.,'\''clean'\'') and '
       .../qa/workunits/ceph-helpers.sh:866: get_num_active_clean:  expression+='not(contains(.,'\''stale'\''))'
       .../qa/workunits/ceph-helpers.sh:867: get_num_active_clean:  expression+=')'
       .../qa/workunits/ceph-helpers.sh:871: get_num_active_clean:  ceph --format xml pg dump pgs
       .../qa/workunits/ceph-helpers.sh:872: get_num_active_clean:  xmlstarlet sel -t -m '//pg_stat/state[(contains(.,'\''active'\'') and contains(.,'\''clean'\'') and not(contains(.,'\''stale'\'')))]' -v . -n
       .../qa/workunits/ceph-helpers.sh:873: get_num_active_clean:  grep -v '^$'
       .../qa/workunits/ceph-helpers.sh:873: get_num_active_clean:  wc -l
       ../qa/workunits/ceph-helpers.sh:993: wait_for_clean:  cur_active_clean=0
       ../qa/workunits/ceph-helpers.sh:994: wait_for_clean:  test 0 '!=' 0
       ../qa/workunits/ceph-helpers.sh:998: wait_for_clean:  sleep 1
       .../qa/workunits/ceph-helpers.sh:999: wait_for_clean:  expr 81 + 1
       ../qa/workunits/ceph-helpers.sh:999: wait_for_clean:  timer=82
       ../qa/workunits/ceph-helpers.sh:985: wait_for_clean:  is_clean
       .../qa/workunits/ceph-helpers.sh:952: is_clean:  get_num_pgs
       .../qa/workunits/ceph-helpers.sh:898: get_num_pgs:  ceph --format xml status
       .../qa/workunits/ceph-helpers.sh:899: get_num_pgs:  xmlstarlet sel -t -m //pgmap/num_pgs -v .
       ../qa/workunits/ceph-helpers.sh:952: is_clean:  num_pgs=4
       ../qa/workunits/ceph-helpers.sh:953: is_clean:  test 4 '!=' 0
       .../qa/workunits/ceph-helpers.sh:954: is_clean:  get_num_active_clean
       .../qa/workunits/ceph-helpers.sh:863: get_num_active_clean:  local 'expression=('
       .../qa/workunits/ceph-helpers.sh:864: get_num_active_clean:  expression+='contains(.,'\''active'\'') and '
       .../qa/workunits/ceph-helpers.sh:865: get_num_active_clean:  expression+='contains(.,'\''clean'\'') and '
       .../qa/workunits/ceph-helpers.sh:866: get_num_active_clean:  expression+='not(contains(.,'\''stale'\''))'
       .../qa/workunits/ceph-helpers.sh:867: get_num_active_clean:  expression+=')'
       .../qa/workunits/ceph-helpers.sh:871: get_num_active_clean:  ceph --format xml pg dump pgs
       .../qa/workunits/ceph-helpers.sh:872: get_num_active_clean:  xmlstarlet sel -t -m '//pg_stat/state[(contains(.,'\''active'\'') and contains(.,'\''clean'\'') and not(contains(.,'\''stale'\'')))]' -v . -n
       .../qa/workunits/ceph-helpers.sh:873: get_num_active_clean:  wc -l
       .../qa/workunits/ceph-helpers.sh:873: get_num_active_clean:  grep -v '^$'
       ../qa/workunits/ceph-helpers.sh:954: is_clean:  test 0 = 4
       ../qa/workunits/ceph-helpers.sh:954: is_clean:  return 1
       ../qa/workunits/ceph-helpers.sh:986: wait_for_clean:  get_is_making_recovery_progress
       .../qa/workunits/ceph-helpers.sh:839: get_is_making_recovery_progress:  ceph --format xml status
       .../qa/workunits/ceph-helpers.sh:839: get_is_making_recovery_progress:  xmlstarlet sel -t -m //pgmap/recovering_keys_per_sec -v . -o ' ' -t -m //pgmap/recovering_bytes_per_sec -v . -o ' ' -t -m //pgmap/recovering_objects_per_sec -v .
       ../qa/workunits/ceph-helpers.sh:839: get_is_making_recovery_progress:  local progress=
       ../qa/workunits/ceph-helpers.sh:840: get_is_making_recovery_progress:  test -n ''
       ../qa/workunits/ceph-helpers.sh:988: wait_for_clean:  ((  timer >= 120  ))
       .../qa/workunits/ceph-helpers.sh:993: wait_for_clean:  get_num_active_clean
       .../qa/workunits/ceph-helpers.sh:863: get_num_active_clean:  local 'expression=('
       .../qa/workunits/ceph-helpers.sh:864: get_num_active_clean:  expression+='contains(.,'\''active'\'') and '
       .../qa/workunits/ceph-helpers.sh:865: get_num_active_clean:  expression+='contains(.,'\''clean'\'') and '
       .../qa/workunits/ceph-helpers.sh:866: get_num_active_clean:  expression+='not(contains(.,'\''stale'\''))'
       .../qa/workunits/ceph-helpers.sh:867: get_num_active_clean:  expression+=')'
       .../qa/workunits/ceph-helpers.sh:871: get_num_active_clean:  ceph --format xml pg dump pgs
       .../qa/workunits/ceph-helpers.sh:872: get_num_active_clean:  xmlstarlet sel -t -m '//pg_stat/state[(contains(.,'\''active'\'') and contains(.,'\''clean'\'') and not(contains(.,'\''stale'\'')))]' -v . -n
       .../qa/workunits/ceph-helpers.sh:873: get_num_active_clean:  grep -v '^$'
       .../qa/workunits/ceph-helpers.sh:873: get_num_active_clean:  wc -l
       ../qa/workunits/ceph-helpers.sh:993: wait_for_clean:  cur_active_clean=0
       ../qa/workunits/ceph-helpers.sh:994: wait_for_clean:  test 0 '!=' 0
       ../qa/workunits/ceph-helpers.sh:998: wait_for_clean:  sleep 1
       .../qa/workunits/ceph-helpers.sh:999: wait_for_clean:  expr 82 + 1
       ../qa/workunits/ceph-helpers.sh:999: wait_for_clean:  timer=83
       ../qa/workunits/ceph-helpers.sh:985: wait_for_clean:  is_clean
       .../qa/workunits/ceph-helpers.sh:952: is_clean:  get_num_pgs
       .../qa/workunits/ceph-helpers.sh:898: get_num_pgs:  ceph --format xml status
       .../qa/workunits/ceph-helpers.sh:899: get_num_pgs:  xmlstarlet sel -t -m //pgmap/num_pgs -v .
       ../qa/workunits/ceph-helpers.sh:952: is_clean:  num_pgs=4
       ../qa/workunits/ceph-helpers.sh:953: is_clean:  test 4 '!=' 0
       .../qa/workunits/ceph-helpers.sh:954: is_clean:  get_num_active_clean
       .../qa/workunits/ceph-helpers.sh:863: get_num_active_clean:  local 'expression=('
       .../qa/workunits/ceph-helpers.sh:864: get_num_active_clean:  expression+='contains(.,'\''active'\'') and '
       .../qa/workunits/ceph-helpers.sh:865: get_num_active_clean:  expression+='contains(.,'\''clean'\'') and '
       .../qa/workunits/ceph-helpers.sh:866: get_num_active_clean:  expression+='not(contains(.,'\''stale'\''))'
       .../qa/workunits/ceph-helpers.sh:867: get_num_active_clean:  expression+=')'
       .../qa/workunits/ceph-helpers.sh:871: get_num_active_clean:  ceph --format xml pg dump pgs
       .../qa/workunits/ceph-helpers.sh:872: get_num_active_clean:  xmlstarlet sel -t -m '//pg_stat/state[(contains(.,'\''active'\'') and contains(.,'\''clean'\'') and not(contains(.,'\''stale'\'')))]' -v . -n
       .../qa/workunits/ceph-helpers.sh:873: get_num_active_clean:  grep -v '^$'
       .../qa/workunits/ceph-helpers.sh:873: get_num_active_clean:  wc -l
       ../qa/workunits/ceph-helpers.sh:954: is_clean:  test 0 = 4
       ../qa/workunits/ceph-helpers.sh:954: is_clean:  return 1
       ../qa/workunits/ceph-helpers.sh:986: wait_for_clean:  get_is_making_recovery_progress
       .../qa/workunits/ceph-helpers.sh:839: get_is_making_recovery_progress:  ceph --format xml status
       .../qa/workunits/ceph-helpers.sh:839: get_is_making_recovery_progress:  xmlstarlet sel -t -m //pgmap/recovering_keys_per_sec -v . -o ' ' -t -m //pgmap/recovering_bytes_per_sec -v . -o ' ' -t -m //pgmap/recovering_objects_per_sec -v .
       ../qa/workunits/ceph-helpers.sh:839: get_is_making_recovery_progress:  local progress=
       ../qa/workunits/ceph-helpers.sh:840: get_is_making_recovery_progress:  test -n ''
       ../qa/workunits/ceph-helpers.sh:988: wait_for_clean:  ((  timer >= 120  ))
       .../qa/workunits/ceph-helpers.sh:993: wait_for_clean:  get_num_active_clean
       .../qa/workunits/ceph-helpers.sh:863: get_num_active_clean:  local 'expression=('
       .../qa/workunits/ceph-helpers.sh:864: get_num_active_clean:  expression+='contains(.,'\''active'\'') and '
       .../qa/workunits/ceph-helpers.sh:865: get_num_active_clean:  expression+='contains(.,'\''clean'\'') and '
       .../qa/workunits/ceph-helpers.sh:866: get_num_active_clean:  expression+='not(contains(.,'\''stale'\''))'
       .../qa/workunits/ceph-helpers.sh:867: get_num_active_clean:  expression+=')'
       .../qa/workunits/ceph-helpers.sh:871: get_num_active_clean:  ceph --format xml pg dump pgs
       .../qa/workunits/ceph-helpers.sh:872: get_num_active_clean:  xmlstarlet sel -t -m '//pg_stat/state[(contains(.,'\''active'\'') and contains(.,'\''clean'\'') and not(contains(.,'\''stale'\'')))]' -v . -n
       .../qa/workunits/ceph-helpers.sh:873: get_num_active_clean:  grep -v '^$'
       .../qa/workunits/ceph-helpers.sh:873: get_num_active_clean:  wc -l
       ../qa/workunits/ceph-helpers.sh:993: wait_for_clean:  cur_active_clean=0
       ../qa/workunits/ceph-helpers.sh:994: wait_for_clean:  test 0 '!=' 0
       ../qa/workunits/ceph-helpers.sh:998: wait_for_clean:  sleep 1
       .../qa/workunits/ceph-helpers.sh:999: wait_for_clean:  expr 83 + 1
       ../qa/workunits/ceph-helpers.sh:999: wait_for_clean:  timer=84
       ../qa/workunits/ceph-helpers.sh:985: wait_for_clean:  is_clean
       .../qa/workunits/ceph-helpers.sh:952: is_clean:  get_num_pgs
       .../qa/workunits/ceph-helpers.sh:898: get_num_pgs:  ceph --format xml status
       .../qa/workunits/ceph-helpers.sh:899: get_num_pgs:  xmlstarlet sel -t -m //pgmap/num_pgs -v .
       ../qa/workunits/ceph-helpers.sh:952: is_clean:  num_pgs=4
       ../qa/workunits/ceph-helpers.sh:953: is_clean:  test 4 '!=' 0
       .../qa/workunits/ceph-helpers.sh:954: is_clean:  get_num_active_clean
       .../qa/workunits/ceph-helpers.sh:863: get_num_active_clean:  local 'expression=('
       .../qa/workunits/ceph-helpers.sh:864: get_num_active_clean:  expression+='contains(.,'\''active'\'') and '
       .../qa/workunits/ceph-helpers.sh:865: get_num_active_clean:  expression+='contains(.,'\''clean'\'') and '
       .../qa/workunits/ceph-helpers.sh:866: get_num_active_clean:  expression+='not(contains(.,'\''stale'\''))'
       .../qa/workunits/ceph-helpers.sh:867: get_num_active_clean:  expression+=')'
       .../qa/workunits/ceph-helpers.sh:871: get_num_active_clean:  ceph --format xml pg dump pgs
       .../qa/workunits/ceph-helpers.sh:872: get_num_active_clean:  xmlstarlet sel -t -m '//pg_stat/state[(contains(.,'\''active'\'') and contains(.,'\''clean'\'') and not(contains(.,'\''stale'\'')))]' -v . -n
       .../qa/workunits/ceph-helpers.sh:873: get_num_active_clean:  grep -v '^$'
       .../qa/workunits/ceph-helpers.sh:873: get_num_active_clean:  wc -l
       ../qa/workunits/ceph-helpers.sh:954: is_clean:  test 0 = 4
       ../qa/workunits/ceph-helpers.sh:954: is_clean:  return 1
       ../qa/workunits/ceph-helpers.sh:986: wait_for_clean:  get_is_making_recovery_progress
       .../qa/workunits/ceph-helpers.sh:839: get_is_making_recovery_progress:  ceph --format xml status
       .../qa/workunits/ceph-helpers.sh:839: get_is_making_recovery_progress:  xmlstarlet sel -t -m //pgmap/recovering_keys_per_sec -v . -o ' ' -t -m //pgmap/recovering_bytes_per_sec -v . -o ' ' -t -m //pgmap/recovering_objects_per_sec -v .
       ../qa/workunits/ceph-helpers.sh:839: get_is_making_recovery_progress:  local progress=
       ../qa/workunits/ceph-helpers.sh:840: get_is_making_recovery_progress:  test -n ''
       ../qa/workunits/ceph-helpers.sh:988: wait_for_clean:  ((  timer >= 120  ))
       .../qa/workunits/ceph-helpers.sh:993: wait_for_clean:  get_num_active_clean
       .../qa/workunits/ceph-helpers.sh:863: get_num_active_clean:  local 'expression=('
       .../qa/workunits/ceph-helpers.sh:864: get_num_active_clean:  expression+='contains(.,'\''active'\'') and '
       .../qa/workunits/ceph-helpers.sh:865: get_num_active_clean:  expression+='contains(.,'\''clean'\'') and '
       .../qa/workunits/ceph-helpers.sh:866: get_num_active_clean:  expression+='not(contains(.,'\''stale'\''))'
       .../qa/workunits/ceph-helpers.sh:867: get_num_active_clean:  expression+=')'
       .../qa/workunits/ceph-helpers.sh:871: get_num_active_clean:  ceph --format xml pg dump pgs
       .../qa/workunits/ceph-helpers.sh:872: get_num_active_clean:  xmlstarlet sel -t -m '//pg_stat/state[(contains(.,'\''active'\'') and contains(.,'\''clean'\'') and not(contains(.,'\''stale'\'')))]' -v . -n
       .../qa/workunits/ceph-helpers.sh:873: get_num_active_clean:  grep -v '^$'
       .../qa/workunits/ceph-helpers.sh:873: get_num_active_clean:  wc -l
       ../qa/workunits/ceph-helpers.sh:993: wait_for_clean:  cur_active_clean=0
       ../qa/workunits/ceph-helpers.sh:994: wait_for_clean:  test 0 '!=' 0
       ../qa/workunits/ceph-helpers.sh:998: wait_for_clean:  sleep 1
       .../qa/workunits/ceph-helpers.sh:999: wait_for_clean:  expr 84 + 1
       ../qa/workunits/ceph-helpers.sh:999: wait_for_clean:  timer=85
       ../qa/workunits/ceph-helpers.sh:985: wait_for_clean:  is_clean
       .../qa/workunits/ceph-helpers.sh:952: is_clean:  get_num_pgs
       .../qa/workunits/ceph-helpers.sh:898: get_num_pgs:  ceph --format xml status
       .../qa/workunits/ceph-helpers.sh:899: get_num_pgs:  xmlstarlet sel -t -m //pgmap/num_pgs -v .
       ../qa/workunits/ceph-helpers.sh:952: is_clean:  num_pgs=4
       ../qa/workunits/ceph-helpers.sh:953: is_clean:  test 4 '!=' 0
       .../qa/workunits/ceph-helpers.sh:954: is_clean:  get_num_active_clean
       .../qa/workunits/ceph-helpers.sh:863: get_num_active_clean:  local 'expression=('
       .../qa/workunits/ceph-helpers.sh:864: get_num_active_clean:  expression+='contains(.,'\''active'\'') and '
       .../qa/workunits/ceph-helpers.sh:865: get_num_active_clean:  expression+='contains(.,'\''clean'\'') and '
       .../qa/workunits/ceph-helpers.sh:866: get_num_active_clean:  expression+='not(contains(.,'\''stale'\''))'
       .../qa/workunits/ceph-helpers.sh:867: get_num_active_clean:  expression+=')'
       .../qa/workunits/ceph-helpers.sh:871: get_num_active_clean:  ceph --format xml pg dump pgs
       .../qa/workunits/ceph-helpers.sh:872: get_num_active_clean:  xmlstarlet sel -t -m '//pg_stat/state[(contains(.,'\''active'\'') and contains(.,'\''clean'\'') and not(contains(.,'\''stale'\'')))]' -v . -n
       .../qa/workunits/ceph-helpers.sh:873: get_num_active_clean:  grep -v '^$'
       .../qa/workunits/ceph-helpers.sh:873: get_num_active_clean:  wc -l
       ../qa/workunits/ceph-helpers.sh:954: is_clean:  test 0 = 4
       ../qa/workunits/ceph-helpers.sh:954: is_clean:  return 1
       ../qa/workunits/ceph-helpers.sh:986: wait_for_clean:  get_is_making_recovery_progress
       .../qa/workunits/ceph-helpers.sh:839: get_is_making_recovery_progress:  ceph --format xml status
       .../qa/workunits/ceph-helpers.sh:839: get_is_making_recovery_progress:  xmlstarlet sel -t -m //pgmap/recovering_keys_per_sec -v . -o ' ' -t -m //pgmap/recovering_bytes_per_sec -v . -o ' ' -t -m //pgmap/recovering_objects_per_sec -v .
       ../qa/workunits/ceph-helpers.sh:839: get_is_making_recovery_progress:  local progress=
       ../qa/workunits/ceph-helpers.sh:840: get_is_making_recovery_progress:  test -n ''
       ../qa/workunits/ceph-helpers.sh:988: wait_for_clean:  ((  timer >= 120  ))
       .../qa/workunits/ceph-helpers.sh:993: wait_for_clean:  get_num_active_clean
       .../qa/workunits/ceph-helpers.sh:863: get_num_active_clean:  local 'expression=('
       .../qa/workunits/ceph-helpers.sh:864: get_num_active_clean:  expression+='contains(.,'\''active'\'') and '
       .../qa/workunits/ceph-helpers.sh:865: get_num_active_clean:  expression+='contains(.,'\''clean'\'') and '
       .../qa/workunits/ceph-helpers.sh:866: get_num_active_clean:  expression+='not(contains(.,'\''stale'\''))'
       .../qa/workunits/ceph-helpers.sh:867: get_num_active_clean:  expression+=')'
       .../qa/workunits/ceph-helpers.sh:871: get_num_active_clean:  ceph --format xml pg dump pgs
       .../qa/workunits/ceph-helpers.sh:872: get_num_active_clean:  xmlstarlet sel -t -m '//pg_stat/state[(contains(.,'\''active'\'') and contains(.,'\''clean'\'') and not(contains(.,'\''stale'\'')))]' -v . -n
       .../qa/workunits/ceph-helpers.sh:873: get_num_active_clean:  grep -v '^$'
       .../qa/workunits/ceph-helpers.sh:873: get_num_active_clean:  wc -l
       ../qa/workunits/ceph-helpers.sh:993: wait_for_clean:  cur_active_clean=0
       ../qa/workunits/ceph-helpers.sh:994: wait_for_clean:  test 0 '!=' 0
       ../qa/workunits/ceph-helpers.sh:998: wait_for_clean:  sleep 1
       .../qa/workunits/ceph-helpers.sh:999: wait_for_clean:  expr 85 + 1
       ../qa/workunits/ceph-helpers.sh:999: wait_for_clean:  timer=86
       ../qa/workunits/ceph-helpers.sh:985: wait_for_clean:  is_clean
       .../qa/workunits/ceph-helpers.sh:952: is_clean:  get_num_pgs
       .../qa/workunits/ceph-helpers.sh:898: get_num_pgs:  ceph --format xml status
       .../qa/workunits/ceph-helpers.sh:899: get_num_pgs:  xmlstarlet sel -t -m //pgmap/num_pgs -v .
       ../qa/workunits/ceph-helpers.sh:952: is_clean:  num_pgs=4
       ../qa/workunits/ceph-helpers.sh:953: is_clean:  test 4 '!=' 0
       .../qa/workunits/ceph-helpers.sh:954: is_clean:  get_num_active_clean
       .../qa/workunits/ceph-helpers.sh:863: get_num_active_clean:  local 'expression=('
       .../qa/workunits/ceph-helpers.sh:864: get_num_active_clean:  expression+='contains(.,'\''active'\'') and '
       .../qa/workunits/ceph-helpers.sh:865: get_num_active_clean:  expression+='contains(.,'\''clean'\'') and '
       .../qa/workunits/ceph-helpers.sh:866: get_num_active_clean:  expression+='not(contains(.,'\''stale'\''))'
       .../qa/workunits/ceph-helpers.sh:867: get_num_active_clean:  expression+=')'
       .../qa/workunits/ceph-helpers.sh:871: get_num_active_clean:  ceph --format xml pg dump pgs
       .../qa/workunits/ceph-helpers.sh:872: get_num_active_clean:  xmlstarlet sel -t -m '//pg_stat/state[(contains(.,'\''active'\'') and contains(.,'\''clean'\'') and not(contains(.,'\''stale'\'')))]' -v . -n
       .../qa/workunits/ceph-helpers.sh:873: get_num_active_clean:  grep -v '^$'
       .../qa/workunits/ceph-helpers.sh:873: get_num_active_clean:  wc -l
       ../qa/workunits/ceph-helpers.sh:954: is_clean:  test 0 = 4
       ../qa/workunits/ceph-helpers.sh:954: is_clean:  return 1
       ../qa/workunits/ceph-helpers.sh:986: wait_for_clean:  get_is_making_recovery_progress
       .../qa/workunits/ceph-helpers.sh:839: get_is_making_recovery_progress:  ceph --format xml status
       .../qa/workunits/ceph-helpers.sh:839: get_is_making_recovery_progress:  xmlstarlet sel -t -m //pgmap/recovering_keys_per_sec -v . -o ' ' -t -m //pgmap/recovering_bytes_per_sec -v . -o ' ' -t -m //pgmap/recovering_objects_per_sec -v .
       ../qa/workunits/ceph-helpers.sh:839: get_is_making_recovery_progress:  local progress=
       ../qa/workunits/ceph-helpers.sh:840: get_is_making_recovery_progress:  test -n ''
       ../qa/workunits/ceph-helpers.sh:988: wait_for_clean:  ((  timer >= 120  ))
       .../qa/workunits/ceph-helpers.sh:993: wait_for_clean:  get_num_active_clean
       .../qa/workunits/ceph-helpers.sh:863: get_num_active_clean:  local 'expression=('
       .../qa/workunits/ceph-helpers.sh:864: get_num_active_clean:  expression+='contains(.,'\''active'\'') and '
       .../qa/workunits/ceph-helpers.sh:865: get_num_active_clean:  expression+='contains(.,'\''clean'\'') and '
       .../qa/workunits/ceph-helpers.sh:866: get_num_active_clean:  expression+='not(contains(.,'\''stale'\''))'
       .../qa/workunits/ceph-helpers.sh:867: get_num_active_clean:  expression+=')'
       .../qa/workunits/ceph-helpers.sh:871: get_num_active_clean:  ceph --format xml pg dump pgs
       .../qa/workunits/ceph-helpers.sh:872: get_num_active_clean:  xmlstarlet sel -t -m '//pg_stat/state[(contains(.,'\''active'\'') and contains(.,'\''clean'\'') and not(contains(.,'\''stale'\'')))]' -v . -n
       .../qa/workunits/ceph-helpers.sh:873: get_num_active_clean:  grep -v '^$'
       .../qa/workunits/ceph-helpers.sh:873: get_num_active_clean:  wc -l
       ../qa/workunits/ceph-helpers.sh:993: wait_for_clean:  cur_active_clean=0
       ../qa/workunits/ceph-helpers.sh:994: wait_for_clean:  test 0 '!=' 0
       ../qa/workunits/ceph-helpers.sh:998: wait_for_clean:  sleep 1
       .../qa/workunits/ceph-helpers.sh:999: wait_for_clean:  expr 86 + 1
       ../qa/workunits/ceph-helpers.sh:999: wait_for_clean:  timer=87
       ../qa/workunits/ceph-helpers.sh:985: wait_for_clean:  is_clean
       .../qa/workunits/ceph-helpers.sh:952: is_clean:  get_num_pgs
       .../qa/workunits/ceph-helpers.sh:898: get_num_pgs:  ceph --format xml status
       .../qa/workunits/ceph-helpers.sh:899: get_num_pgs:  xmlstarlet sel -t -m //pgmap/num_pgs -v .
       ../qa/workunits/ceph-helpers.sh:952: is_clean:  num_pgs=4
       ../qa/workunits/ceph-helpers.sh:953: is_clean:  test 4 '!=' 0
       .../qa/workunits/ceph-helpers.sh:954: is_clean:  get_num_active_clean
       .../qa/workunits/ceph-helpers.sh:863: get_num_active_clean:  local 'expression=('
       .../qa/workunits/ceph-helpers.sh:864: get_num_active_clean:  expression+='contains(.,'\''active'\'') and '
       .../qa/workunits/ceph-helpers.sh:865: get_num_active_clean:  expression+='contains(.,'\''clean'\'') and '
       .../qa/workunits/ceph-helpers.sh:866: get_num_active_clean:  expression+='not(contains(.,'\''stale'\''))'
       .../qa/workunits/ceph-helpers.sh:867: get_num_active_clean:  expression+=')'
       .../qa/workunits/ceph-helpers.sh:871: get_num_active_clean:  ceph --format xml pg dump pgs
       .../qa/workunits/ceph-helpers.sh:872: get_num_active_clean:  xmlstarlet sel -t -m '//pg_stat/state[(contains(.,'\''active'\'') and contains(.,'\''clean'\'') and not(contains(.,'\''stale'\'')))]' -v . -n
       .../qa/workunits/ceph-helpers.sh:873: get_num_active_clean:  wc -l
       .../qa/workunits/ceph-helpers.sh:873: get_num_active_clean:  grep -v '^$'
       ../qa/workunits/ceph-helpers.sh:954: is_clean:  test 0 = 4
       ../qa/workunits/ceph-helpers.sh:954: is_clean:  return 1
       ../qa/workunits/ceph-helpers.sh:986: wait_for_clean:  get_is_making_recovery_progress
       .../qa/workunits/ceph-helpers.sh:839: get_is_making_recovery_progress:  ceph --format xml status
       .../qa/workunits/ceph-helpers.sh:839: get_is_making_recovery_progress:  xmlstarlet sel -t -m //pgmap/recovering_keys_per_sec -v . -o ' ' -t -m //pgmap/recovering_bytes_per_sec -v . -o ' ' -t -m //pgmap/recovering_objects_per_sec -v .
       ../qa/workunits/ceph-helpers.sh:839: get_is_making_recovery_progress:  local progress=
       ../qa/workunits/ceph-helpers.sh:840: get_is_making_recovery_progress:  test -n ''
       ../qa/workunits/ceph-helpers.sh:988: wait_for_clean:  ((  timer >= 120  ))
       .../qa/workunits/ceph-helpers.sh:993: wait_for_clean:  get_num_active_clean
       .../qa/workunits/ceph-helpers.sh:863: get_num_active_clean:  local 'expression=('
       .../qa/workunits/ceph-helpers.sh:864: get_num_active_clean:  expression+='contains(.,'\''active'\'') and '
       .../qa/workunits/ceph-helpers.sh:865: get_num_active_clean:  expression+='contains(.,'\''clean'\'') and '
       .../qa/workunits/ceph-helpers.sh:866: get_num_active_clean:  expression+='not(contains(.,'\''stale'\''))'
       .../qa/workunits/ceph-helpers.sh:867: get_num_active_clean:  expression+=')'
       .../qa/workunits/ceph-helpers.sh:871: get_num_active_clean:  ceph --format xml pg dump pgs
       .../qa/workunits/ceph-helpers.sh:872: get_num_active_clean:  xmlstarlet sel -t -m '//pg_stat/state[(contains(.,'\''active'\'') and contains(.,'\''clean'\'') and not(contains(.,'\''stale'\'')))]' -v . -n
       .../qa/workunits/ceph-helpers.sh:873: get_num_active_clean:  grep -v '^$'
       .../qa/workunits/ceph-helpers.sh:873: get_num_active_clean:  wc -l
       ../qa/workunits/ceph-helpers.sh:993: wait_for_clean:  cur_active_clean=0
       ../qa/workunits/ceph-helpers.sh:994: wait_for_clean:  test 0 '!=' 0
       ../qa/workunits/ceph-helpers.sh:998: wait_for_clean:  sleep 1
       .../qa/workunits/ceph-helpers.sh:999: wait_for_clean:  expr 87 + 1
       ../qa/workunits/ceph-helpers.sh:999: wait_for_clean:  timer=88
       ../qa/workunits/ceph-helpers.sh:985: wait_for_clean:  is_clean
       .../qa/workunits/ceph-helpers.sh:952: is_clean:  get_num_pgs
       .../qa/workunits/ceph-helpers.sh:898: get_num_pgs:  ceph --format xml status
       .../qa/workunits/ceph-helpers.sh:899: get_num_pgs:  xmlstarlet sel -t -m //pgmap/num_pgs -v .
       ../qa/workunits/ceph-helpers.sh:952: is_clean:  num_pgs=4
       ../qa/workunits/ceph-helpers.sh:953: is_clean:  test 4 '!=' 0
       .../qa/workunits/ceph-helpers.sh:954: is_clean:  get_num_active_clean
       .../qa/workunits/ceph-helpers.sh:863: get_num_active_clean:  local 'expression=('
       .../qa/workunits/ceph-helpers.sh:864: get_num_active_clean:  expression+='contains(.,'\''active'\'') and '
       .../qa/workunits/ceph-helpers.sh:865: get_num_active_clean:  expression+='contains(.,'\''clean'\'') and '
       .../qa/workunits/ceph-helpers.sh:866: get_num_active_clean:  expression+='not(contains(.,'\''stale'\''))'
       .../qa/workunits/ceph-helpers.sh:867: get_num_active_clean:  expression+=')'
       .../qa/workunits/ceph-helpers.sh:871: get_num_active_clean:  ceph --format xml pg dump pgs
       .../qa/workunits/ceph-helpers.sh:872: get_num_active_clean:  xmlstarlet sel -t -m '//pg_stat/state[(contains(.,'\''active'\'') and contains(.,'\''clean'\'') and not(contains(.,'\''stale'\'')))]' -v . -n
       .../qa/workunits/ceph-helpers.sh:873: get_num_active_clean:  grep -v '^$'
       .../qa/workunits/ceph-helpers.sh:873: get_num_active_clean:  wc -l
       ../qa/workunits/ceph-helpers.sh:954: is_clean:  test 0 = 4
       ../qa/workunits/ceph-helpers.sh:954: is_clean:  return 1
       ../qa/workunits/ceph-helpers.sh:986: wait_for_clean:  get_is_making_recovery_progress
       .../qa/workunits/ceph-helpers.sh:839: get_is_making_recovery_progress:  ceph --format xml status
       .../qa/workunits/ceph-helpers.sh:839: get_is_making_recovery_progress:  xmlstarlet sel -t -m //pgmap/recovering_keys_per_sec -v . -o ' ' -t -m //pgmap/recovering_bytes_per_sec -v . -o ' ' -t -m //pgmap/recovering_objects_per_sec -v .
       ../qa/workunits/ceph-helpers.sh:839: get_is_making_recovery_progress:  local progress=
       ../qa/workunits/ceph-helpers.sh:840: get_is_making_recovery_progress:  test -n ''
       ../qa/workunits/ceph-helpers.sh:988: wait_for_clean:  ((  timer >= 120  ))
       .../qa/workunits/ceph-helpers.sh:993: wait_for_clean:  get_num_active_clean
       .../qa/workunits/ceph-helpers.sh:863: get_num_active_clean:  local 'expression=('
       .../qa/workunits/ceph-helpers.sh:864: get_num_active_clean:  expression+='contains(.,'\''active'\'') and '
       .../qa/workunits/ceph-helpers.sh:865: get_num_active_clean:  expression+='contains(.,'\''clean'\'') and '
       .../qa/workunits/ceph-helpers.sh:866: get_num_active_clean:  expression+='not(contains(.,'\''stale'\''))'
       .../qa/workunits/ceph-helpers.sh:867: get_num_active_clean:  expression+=')'
       .../qa/workunits/ceph-helpers.sh:871: get_num_active_clean:  ceph --format xml pg dump pgs
       .../qa/workunits/ceph-helpers.sh:872: get_num_active_clean:  xmlstarlet sel -t -m '//pg_stat/state[(contains(.,'\''active'\'') and contains(.,'\''clean'\'') and not(contains(.,'\''stale'\'')))]' -v . -n
       .../qa/workunits/ceph-helpers.sh:873: get_num_active_clean:  grep -v '^$'
       .../qa/workunits/ceph-helpers.sh:873: get_num_active_clean:  wc -l
       ../qa/workunits/ceph-helpers.sh:993: wait_for_clean:  cur_active_clean=0
       ../qa/workunits/ceph-helpers.sh:994: wait_for_clean:  test 0 '!=' 0
       ../qa/workunits/ceph-helpers.sh:998: wait_for_clean:  sleep 1
       .../qa/workunits/ceph-helpers.sh:999: wait_for_clean:  expr 88 + 1
       ../qa/workunits/ceph-helpers.sh:999: wait_for_clean:  timer=89
       ../qa/workunits/ceph-helpers.sh:985: wait_for_clean:  is_clean
       .../qa/workunits/ceph-helpers.sh:952: is_clean:  get_num_pgs
       .../qa/workunits/ceph-helpers.sh:898: get_num_pgs:  ceph --format xml status
       .../qa/workunits/ceph-helpers.sh:899: get_num_pgs:  xmlstarlet sel -t -m //pgmap/num_pgs -v .
       ../qa/workunits/ceph-helpers.sh:952: is_clean:  num_pgs=4
       ../qa/workunits/ceph-helpers.sh:953: is_clean:  test 4 '!=' 0
       .../qa/workunits/ceph-helpers.sh:954: is_clean:  get_num_active_clean
       .../qa/workunits/ceph-helpers.sh:863: get_num_active_clean:  local 'expression=('
       .../qa/workunits/ceph-helpers.sh:864: get_num_active_clean:  expression+='contains(.,'\''active'\'') and '
       .../qa/workunits/ceph-helpers.sh:865: get_num_active_clean:  expression+='contains(.,'\''clean'\'') and '
       .../qa/workunits/ceph-helpers.sh:866: get_num_active_clean:  expression+='not(contains(.,'\''stale'\''))'
       .../qa/workunits/ceph-helpers.sh:867: get_num_active_clean:  expression+=')'
       .../qa/workunits/ceph-helpers.sh:871: get_num_active_clean:  ceph --format xml pg dump pgs
       .../qa/workunits/ceph-helpers.sh:872: get_num_active_clean:  xmlstarlet sel -t -m '//pg_stat/state[(contains(.,'\''active'\'') and contains(.,'\''clean'\'') and not(contains(.,'\''stale'\'')))]' -v . -n
       .../qa/workunits/ceph-helpers.sh:873: get_num_active_clean:  grep -v '^$'
       .../qa/workunits/ceph-helpers.sh:873: get_num_active_clean:  wc -l
       ../qa/workunits/ceph-helpers.sh:954: is_clean:  test 0 = 4
       ../qa/workunits/ceph-helpers.sh:954: is_clean:  return 1
       ../qa/workunits/ceph-helpers.sh:986: wait_for_clean:  get_is_making_recovery_progress
       .../qa/workunits/ceph-helpers.sh:839: get_is_making_recovery_progress:  ceph --format xml status
       .../qa/workunits/ceph-helpers.sh:839: get_is_making_recovery_progress:  xmlstarlet sel -t -m //pgmap/recovering_keys_per_sec -v . -o ' ' -t -m //pgmap/recovering_bytes_per_sec -v . -o ' ' -t -m //pgmap/recovering_objects_per_sec -v .
       ../qa/workunits/ceph-helpers.sh:839: get_is_making_recovery_progress:  local progress=
       ../qa/workunits/ceph-helpers.sh:840: get_is_making_recovery_progress:  test -n ''
       ../qa/workunits/ceph-helpers.sh:988: wait_for_clean:  ((  timer >= 120  ))
       .../qa/workunits/ceph-helpers.sh:993: wait_for_clean:  get_num_active_clean
       .../qa/workunits/ceph-helpers.sh:863: get_num_active_clean:  local 'expression=('
       .../qa/workunits/ceph-helpers.sh:864: get_num_active_clean:  expression+='contains(.,'\''active'\'') and '
       .../qa/workunits/ceph-helpers.sh:865: get_num_active_clean:  expression+='contains(.,'\''clean'\'') and '
       .../qa/workunits/ceph-helpers.sh:866: get_num_active_clean:  expression+='not(contains(.,'\''stale'\''))'
       .../qa/workunits/ceph-helpers.sh:867: get_num_active_clean:  expression+=')'
       .../qa/workunits/ceph-helpers.sh:871: get_num_active_clean:  ceph --format xml pg dump pgs
       .../qa/workunits/ceph-helpers.sh:872: get_num_active_clean:  xmlstarlet sel -t -m '//pg_stat/state[(contains(.,'\''active'\'') and contains(.,'\''clean'\'') and not(contains(.,'\''stale'\'')))]' -v . -n
       .../qa/workunits/ceph-helpers.sh:873: get_num_active_clean:  grep -v '^$'
       .../qa/workunits/ceph-helpers.sh:873: get_num_active_clean:  wc -l
       ../qa/workunits/ceph-helpers.sh:993: wait_for_clean:  cur_active_clean=0
       ../qa/workunits/ceph-helpers.sh:994: wait_for_clean:  test 0 '!=' 0
       ../qa/workunits/ceph-helpers.sh:998: wait_for_clean:  sleep 1
       .../qa/workunits/ceph-helpers.sh:999: wait_for_clean:  expr 89 + 1
       ../qa/workunits/ceph-helpers.sh:999: wait_for_clean:  timer=90
       ../qa/workunits/ceph-helpers.sh:985: wait_for_clean:  is_clean
       .../qa/workunits/ceph-helpers.sh:952: is_clean:  get_num_pgs
       .../qa/workunits/ceph-helpers.sh:898: get_num_pgs:  ceph --format xml status
       .../qa/workunits/ceph-helpers.sh:899: get_num_pgs:  xmlstarlet sel -t -m //pgmap/num_pgs -v .
       ../qa/workunits/ceph-helpers.sh:952: is_clean:  num_pgs=4
       ../qa/workunits/ceph-helpers.sh:953: is_clean:  test 4 '!=' 0
       .../qa/workunits/ceph-helpers.sh:954: is_clean:  get_num_active_clean
       .../qa/workunits/ceph-helpers.sh:863: get_num_active_clean:  local 'expression=('
       .../qa/workunits/ceph-helpers.sh:864: get_num_active_clean:  expression+='contains(.,'\''active'\'') and '
       .../qa/workunits/ceph-helpers.sh:865: get_num_active_clean:  expression+='contains(.,'\''clean'\'') and '
       .../qa/workunits/ceph-helpers.sh:866: get_num_active_clean:  expression+='not(contains(.,'\''stale'\''))'
       .../qa/workunits/ceph-helpers.sh:867: get_num_active_clean:  expression+=')'
       .../qa/workunits/ceph-helpers.sh:871: get_num_active_clean:  ceph --format xml pg dump pgs
       .../qa/workunits/ceph-helpers.sh:872: get_num_active_clean:  xmlstarlet sel -t -m '//pg_stat/state[(contains(.,'\''active'\'') and contains(.,'\''clean'\'') and not(contains(.,'\''stale'\'')))]' -v . -n
       .../qa/workunits/ceph-helpers.sh:873: get_num_active_clean:  grep -v '^$'
       .../qa/workunits/ceph-helpers.sh:873: get_num_active_clean:  wc -l
       ../qa/workunits/ceph-helpers.sh:954: is_clean:  test 0 = 4
       ../qa/workunits/ceph-helpers.sh:954: is_clean:  return 1
       ../qa/workunits/ceph-helpers.sh:986: wait_for_clean:  get_is_making_recovery_progress
       .../qa/workunits/ceph-helpers.sh:839: get_is_making_recovery_progress:  ceph --format xml status
       .../qa/workunits/ceph-helpers.sh:839: get_is_making_recovery_progress:  xmlstarlet sel -t -m //pgmap/recovering_keys_per_sec -v . -o ' ' -t -m //pgmap/recovering_bytes_per_sec -v . -o ' ' -t -m //pgmap/recovering_objects_per_sec -v .
       ../qa/workunits/ceph-helpers.sh:839: get_is_making_recovery_progress:  local progress=
       ../qa/workunits/ceph-helpers.sh:840: get_is_making_recovery_progress:  test -n ''
       ../qa/workunits/ceph-helpers.sh:988: wait_for_clean:  ((  timer >= 120  ))
       .../qa/workunits/ceph-helpers.sh:993: wait_for_clean:  get_num_active_clean
       .../qa/workunits/ceph-helpers.sh:863: get_num_active_clean:  local 'expression=('
       .../qa/workunits/ceph-helpers.sh:864: get_num_active_clean:  expression+='contains(.,'\''active'\'') and '
       .../qa/workunits/ceph-helpers.sh:865: get_num_active_clean:  expression+='contains(.,'\''clean'\'') and '
       .../qa/workunits/ceph-helpers.sh:866: get_num_active_clean:  expression+='not(contains(.,'\''stale'\''))'
       .../qa/workunits/ceph-helpers.sh:867: get_num_active_clean:  expression+=')'
       .../qa/workunits/ceph-helpers.sh:871: get_num_active_clean:  ceph --format xml pg dump pgs
       .../qa/workunits/ceph-helpers.sh:872: get_num_active_clean:  xmlstarlet sel -t -m '//pg_stat/state[(contains(.,'\''active'\'') and contains(.,'\''clean'\'') and not(contains(.,'\''stale'\'')))]' -v . -n
       .../qa/workunits/ceph-helpers.sh:873: get_num_active_clean:  grep -v '^$'
       .../qa/workunits/ceph-helpers.sh:873: get_num_active_clean:  wc -l
       ../qa/workunits/ceph-helpers.sh:993: wait_for_clean:  cur_active_clean=0
       ../qa/workunits/ceph-helpers.sh:994: wait_for_clean:  test 0 '!=' 0
       ../qa/workunits/ceph-helpers.sh:998: wait_for_clean:  sleep 1
       .../qa/workunits/ceph-helpers.sh:999: wait_for_clean:  expr 90 + 1
       ../qa/workunits/ceph-helpers.sh:999: wait_for_clean:  timer=91
       ../qa/workunits/ceph-helpers.sh:985: wait_for_clean:  is_clean
       .../qa/workunits/ceph-helpers.sh:952: is_clean:  get_num_pgs
       .../qa/workunits/ceph-helpers.sh:898: get_num_pgs:  ceph --format xml status
       .../qa/workunits/ceph-helpers.sh:899: get_num_pgs:  xmlstarlet sel -t -m //pgmap/num_pgs -v .
       ../qa/workunits/ceph-helpers.sh:952: is_clean:  num_pgs=4
       ../qa/workunits/ceph-helpers.sh:953: is_clean:  test 4 '!=' 0
       .../qa/workunits/ceph-helpers.sh:954: is_clean:  get_num_active_clean
       .../qa/workunits/ceph-helpers.sh:863: get_num_active_clean:  local 'expression=('
       .../qa/workunits/ceph-helpers.sh:864: get_num_active_clean:  expression+='contains(.,'\''active'\'') and '
       .../qa/workunits/ceph-helpers.sh:865: get_num_active_clean:  expression+='contains(.,'\''clean'\'') and '
       .../qa/workunits/ceph-helpers.sh:866: get_num_active_clean:  expression+='not(contains(.,'\''stale'\''))'
       .../qa/workunits/ceph-helpers.sh:867: get_num_active_clean:  expression+=')'
       .../qa/workunits/ceph-helpers.sh:871: get_num_active_clean:  ceph --format xml pg dump pgs
       .../qa/workunits/ceph-helpers.sh:872: get_num_active_clean:  xmlstarlet sel -t -m '//pg_stat/state[(contains(.,'\''active'\'') and contains(.,'\''clean'\'') and not(contains(.,'\''stale'\'')))]' -v . -n
       .../qa/workunits/ceph-helpers.sh:873: get_num_active_clean:  grep -v '^$'
       .../qa/workunits/ceph-helpers.sh:873: get_num_active_clean:  wc -l
       ../qa/workunits/ceph-helpers.sh:954: is_clean:  test 0 = 4
       ../qa/workunits/ceph-helpers.sh:954: is_clean:  return 1
       ../qa/workunits/ceph-helpers.sh:986: wait_for_clean:  get_is_making_recovery_progress
       .../qa/workunits/ceph-helpers.sh:839: get_is_making_recovery_progress:  ceph --format xml status
       .../qa/workunits/ceph-helpers.sh:839: get_is_making_recovery_progress:  xmlstarlet sel -t -m //pgmap/recovering_keys_per_sec -v . -o ' ' -t -m //pgmap/recovering_bytes_per_sec -v . -o ' ' -t -m //pgmap/recovering_objects_per_sec -v .
       ../qa/workunits/ceph-helpers.sh:839: get_is_making_recovery_progress:  local progress=
       ../qa/workunits/ceph-helpers.sh:840: get_is_making_recovery_progress:  test -n ''
       ../qa/workunits/ceph-helpers.sh:988: wait_for_clean:  ((  timer >= 120  ))
       .../qa/workunits/ceph-helpers.sh:993: wait_for_clean:  get_num_active_clean
       .../qa/workunits/ceph-helpers.sh:863: get_num_active_clean:  local 'expression=('
       .../qa/workunits/ceph-helpers.sh:864: get_num_active_clean:  expression+='contains(.,'\''active'\'') and '
       .../qa/workunits/ceph-helpers.sh:865: get_num_active_clean:  expression+='contains(.,'\''clean'\'') and '
       .../qa/workunits/ceph-helpers.sh:866: get_num_active_clean:  expression+='not(contains(.,'\''stale'\''))'
       .../qa/workunits/ceph-helpers.sh:867: get_num_active_clean:  expression+=')'
       .../qa/workunits/ceph-helpers.sh:871: get_num_active_clean:  ceph --format xml pg dump pgs
       .../qa/workunits/ceph-helpers.sh:872: get_num_active_clean:  xmlstarlet sel -t -m '//pg_stat/state[(contains(.,'\''active'\'') and contains(.,'\''clean'\'') and not(contains(.,'\''stale'\'')))]' -v . -n
       .../qa/workunits/ceph-helpers.sh:873: get_num_active_clean:  grep -v '^$'
       .../qa/workunits/ceph-helpers.sh:873: get_num_active_clean:  wc -l
       ../qa/workunits/ceph-helpers.sh:993: wait_for_clean:  cur_active_clean=0
       ../qa/workunits/ceph-helpers.sh:994: wait_for_clean:  test 0 '!=' 0
       ../qa/workunits/ceph-helpers.sh:998: wait_for_clean:  sleep 1
       .../qa/workunits/ceph-helpers.sh:999: wait_for_clean:  expr 91 + 1
       ../qa/workunits/ceph-helpers.sh:999: wait_for_clean:  timer=92
       ../qa/workunits/ceph-helpers.sh:985: wait_for_clean:  is_clean
       .../qa/workunits/ceph-helpers.sh:952: is_clean:  get_num_pgs
       .../qa/workunits/ceph-helpers.sh:898: get_num_pgs:  ceph --format xml status
       .../qa/workunits/ceph-helpers.sh:899: get_num_pgs:  xmlstarlet sel -t -m //pgmap/num_pgs -v .
       ../qa/workunits/ceph-helpers.sh:952: is_clean:  num_pgs=4
       ../qa/workunits/ceph-helpers.sh:953: is_clean:  test 4 '!=' 0
       .../qa/workunits/ceph-helpers.sh:954: is_clean:  get_num_active_clean
       .../qa/workunits/ceph-helpers.sh:863: get_num_active_clean:  local 'expression=('
       .../qa/workunits/ceph-helpers.sh:864: get_num_active_clean:  expression+='contains(.,'\''active'\'') and '
       .../qa/workunits/ceph-helpers.sh:865: get_num_active_clean:  expression+='contains(.,'\''clean'\'') and '
       .../qa/workunits/ceph-helpers.sh:866: get_num_active_clean:  expression+='not(contains(.,'\''stale'\''))'
       .../qa/workunits/ceph-helpers.sh:867: get_num_active_clean:  expression+=')'
       .../qa/workunits/ceph-helpers.sh:871: get_num_active_clean:  ceph --format xml pg dump pgs
       .../qa/workunits/ceph-helpers.sh:872: get_num_active_clean:  xmlstarlet sel -t -m '//pg_stat/state[(contains(.,'\''active'\'') and contains(.,'\''clean'\'') and not(contains(.,'\''stale'\'')))]' -v . -n
       .../qa/workunits/ceph-helpers.sh:873: get_num_active_clean:  wc -l
       .../qa/workunits/ceph-helpers.sh:873: get_num_active_clean:  grep -v '^$'
       ../qa/workunits/ceph-helpers.sh:954: is_clean:  test 0 = 4
       ../qa/workunits/ceph-helpers.sh:954: is_clean:  return 1
       ../qa/workunits/ceph-helpers.sh:986: wait_for_clean:  get_is_making_recovery_progress
       .../qa/workunits/ceph-helpers.sh:839: get_is_making_recovery_progress:  ceph --format xml status
       .../qa/workunits/ceph-helpers.sh:839: get_is_making_recovery_progress:  xmlstarlet sel -t -m //pgmap/recovering_keys_per_sec -v . -o ' ' -t -m //pgmap/recovering_bytes_per_sec -v . -o ' ' -t -m //pgmap/recovering_objects_per_sec -v .
       ../qa/workunits/ceph-helpers.sh:839: get_is_making_recovery_progress:  local progress=
       ../qa/workunits/ceph-helpers.sh:840: get_is_making_recovery_progress:  test -n ''
       ../qa/workunits/ceph-helpers.sh:988: wait_for_clean:  ((  timer >= 120  ))
       .../qa/workunits/ceph-helpers.sh:993: wait_for_clean:  get_num_active_clean
       .../qa/workunits/ceph-helpers.sh:863: get_num_active_clean:  local 'expression=('
       .../qa/workunits/ceph-helpers.sh:864: get_num_active_clean:  expression+='contains(.,'\''active'\'') and '
       .../qa/workunits/ceph-helpers.sh:865: get_num_active_clean:  expression+='contains(.,'\''clean'\'') and '
       .../qa/workunits/ceph-helpers.sh:866: get_num_active_clean:  expression+='not(contains(.,'\''stale'\''))'
       .../qa/workunits/ceph-helpers.sh:867: get_num_active_clean:  expression+=')'
       .../qa/workunits/ceph-helpers.sh:871: get_num_active_clean:  ceph --format xml pg dump pgs
       .../qa/workunits/ceph-helpers.sh:872: get_num_active_clean:  xmlstarlet sel -t -m '//pg_stat/state[(contains(.,'\''active'\'') and contains(.,'\''clean'\'') and not(contains(.,'\''stale'\'')))]' -v . -n
       .../qa/workunits/ceph-helpers.sh:873: get_num_active_clean:  grep -v '^$'
       .../qa/workunits/ceph-helpers.sh:873: get_num_active_clean:  wc -l
       ../qa/workunits/ceph-helpers.sh:993: wait_for_clean:  cur_active_clean=0
       ../qa/workunits/ceph-helpers.sh:994: wait_for_clean:  test 0 '!=' 0
       ../qa/workunits/ceph-helpers.sh:998: wait_for_clean:  sleep 1
       .../qa/workunits/ceph-helpers.sh:999: wait_for_clean:  expr 92 + 1
       ../qa/workunits/ceph-helpers.sh:999: wait_for_clean:  timer=93
       ../qa/workunits/ceph-helpers.sh:985: wait_for_clean:  is_clean
       .../qa/workunits/ceph-helpers.sh:952: is_clean:  get_num_pgs
       .../qa/workunits/ceph-helpers.sh:898: get_num_pgs:  ceph --format xml status
       .../qa/workunits/ceph-helpers.sh:899: get_num_pgs:  xmlstarlet sel -t -m //pgmap/num_pgs -v .
       ../qa/workunits/ceph-helpers.sh:952: is_clean:  num_pgs=4
       ../qa/workunits/ceph-helpers.sh:953: is_clean:  test 4 '!=' 0
       .../qa/workunits/ceph-helpers.sh:954: is_clean:  get_num_active_clean
       .../qa/workunits/ceph-helpers.sh:863: get_num_active_clean:  local 'expression=('
       .../qa/workunits/ceph-helpers.sh:864: get_num_active_clean:  expression+='contains(.,'\''active'\'') and '
       .../qa/workunits/ceph-helpers.sh:865: get_num_active_clean:  expression+='contains(.,'\''clean'\'') and '
       .../qa/workunits/ceph-helpers.sh:866: get_num_active_clean:  expression+='not(contains(.,'\''stale'\''))'
       .../qa/workunits/ceph-helpers.sh:867: get_num_active_clean:  expression+=')'
       .../qa/workunits/ceph-helpers.sh:871: get_num_active_clean:  ceph --format xml pg dump pgs
       .../qa/workunits/ceph-helpers.sh:872: get_num_active_clean:  xmlstarlet sel -t -m '//pg_stat/state[(contains(.,'\''active'\'') and contains(.,'\''clean'\'') and not(contains(.,'\''stale'\'')))]' -v . -n
       .../qa/workunits/ceph-helpers.sh:873: get_num_active_clean:  grep -v '^$'
       .../qa/workunits/ceph-helpers.sh:873: get_num_active_clean:  wc -l
       ../qa/workunits/ceph-helpers.sh:954: is_clean:  test 0 = 4
       ../qa/workunits/ceph-helpers.sh:954: is_clean:  return 1
       ../qa/workunits/ceph-helpers.sh:986: wait_for_clean:  get_is_making_recovery_progress
       .../qa/workunits/ceph-helpers.sh:839: get_is_making_recovery_progress:  ceph --format xml status
       .../qa/workunits/ceph-helpers.sh:839: get_is_making_recovery_progress:  xmlstarlet sel -t -m //pgmap/recovering_keys_per_sec -v . -o ' ' -t -m //pgmap/recovering_bytes_per_sec -v . -o ' ' -t -m //pgmap/recovering_objects_per_sec -v .
       ../qa/workunits/ceph-helpers.sh:839: get_is_making_recovery_progress:  local progress=
       ../qa/workunits/ceph-helpers.sh:840: get_is_making_recovery_progress:  test -n ''
       ../qa/workunits/ceph-helpers.sh:988: wait_for_clean:  ((  timer >= 120  ))
       .../qa/workunits/ceph-helpers.sh:993: wait_for_clean:  get_num_active_clean
       .../qa/workunits/ceph-helpers.sh:863: get_num_active_clean:  local 'expression=('
       .../qa/workunits/ceph-helpers.sh:864: get_num_active_clean:  expression+='contains(.,'\''active'\'') and '
       .../qa/workunits/ceph-helpers.sh:865: get_num_active_clean:  expression+='contains(.,'\''clean'\'') and '
       .../qa/workunits/ceph-helpers.sh:866: get_num_active_clean:  expression+='not(contains(.,'\''stale'\''))'
       .../qa/workunits/ceph-helpers.sh:867: get_num_active_clean:  expression+=')'
       .../qa/workunits/ceph-helpers.sh:871: get_num_active_clean:  ceph --format xml pg dump pgs
       .../qa/workunits/ceph-helpers.sh:872: get_num_active_clean:  xmlstarlet sel -t -m '//pg_stat/state[(contains(.,'\''active'\'') and contains(.,'\''clean'\'') and not(contains(.,'\''stale'\'')))]' -v . -n
       .../qa/workunits/ceph-helpers.sh:873: get_num_active_clean:  grep -v '^$'
       .../qa/workunits/ceph-helpers.sh:873: get_num_active_clean:  wc -l
       ../qa/workunits/ceph-helpers.sh:993: wait_for_clean:  cur_active_clean=0
       ../qa/workunits/ceph-helpers.sh:994: wait_for_clean:  test 0 '!=' 0
       ../qa/workunits/ceph-helpers.sh:998: wait_for_clean:  sleep 1
       .../qa/workunits/ceph-helpers.sh:999: wait_for_clean:  expr 93 + 1
       ../qa/workunits/ceph-helpers.sh:999: wait_for_clean:  timer=94
       ../qa/workunits/ceph-helpers.sh:985: wait_for_clean:  is_clean
       .../qa/workunits/ceph-helpers.sh:952: is_clean:  get_num_pgs
       .../qa/workunits/ceph-helpers.sh:898: get_num_pgs:  ceph --format xml status
       .../qa/workunits/ceph-helpers.sh:899: get_num_pgs:  xmlstarlet sel -t -m //pgmap/num_pgs -v .
       ../qa/workunits/ceph-helpers.sh:952: is_clean:  num_pgs=4
       ../qa/workunits/ceph-helpers.sh:953: is_clean:  test 4 '!=' 0
       .../qa/workunits/ceph-helpers.sh:954: is_clean:  get_num_active_clean
       .../qa/workunits/ceph-helpers.sh:863: get_num_active_clean:  local 'expression=('
       .../qa/workunits/ceph-helpers.sh:864: get_num_active_clean:  expression+='contains(.,'\''active'\'') and '
       .../qa/workunits/ceph-helpers.sh:865: get_num_active_clean:  expression+='contains(.,'\''clean'\'') and '
       .../qa/workunits/ceph-helpers.sh:866: get_num_active_clean:  expression+='not(contains(.,'\''stale'\''))'
       .../qa/workunits/ceph-helpers.sh:867: get_num_active_clean:  expression+=')'
       .../qa/workunits/ceph-helpers.sh:871: get_num_active_clean:  ceph --format xml pg dump pgs
       .../qa/workunits/ceph-helpers.sh:872: get_num_active_clean:  xmlstarlet sel -t -m '//pg_stat/state[(contains(.,'\''active'\'') and contains(.,'\''clean'\'') and not(contains(.,'\''stale'\'')))]' -v . -n
       .../qa/workunits/ceph-helpers.sh:873: get_num_active_clean:  wc -l
       .../qa/workunits/ceph-helpers.sh:873: get_num_active_clean:  grep -v '^$'
       ../qa/workunits/ceph-helpers.sh:954: is_clean:  test 0 = 4
       ../qa/workunits/ceph-helpers.sh:954: is_clean:  return 1
       ../qa/workunits/ceph-helpers.sh:986: wait_for_clean:  get_is_making_recovery_progress
       .../qa/workunits/ceph-helpers.sh:839: get_is_making_recovery_progress:  ceph --format xml status
       .../qa/workunits/ceph-helpers.sh:839: get_is_making_recovery_progress:  xmlstarlet sel -t -m //pgmap/recovering_keys_per_sec -v . -o ' ' -t -m //pgmap/recovering_bytes_per_sec -v . -o ' ' -t -m //pgmap/recovering_objects_per_sec -v .
       ../qa/workunits/ceph-helpers.sh:839: get_is_making_recovery_progress:  local progress=
       ../qa/workunits/ceph-helpers.sh:840: get_is_making_recovery_progress:  test -n ''
       ../qa/workunits/ceph-helpers.sh:988: wait_for_clean:  ((  timer >= 120  ))
       .../qa/workunits/ceph-helpers.sh:993: wait_for_clean:  get_num_active_clean
       .../qa/workunits/ceph-helpers.sh:863: get_num_active_clean:  local 'expression=('
       .../qa/workunits/ceph-helpers.sh:864: get_num_active_clean:  expression+='contains(.,'\''active'\'') and '
       .../qa/workunits/ceph-helpers.sh:865: get_num_active_clean:  expression+='contains(.,'\''clean'\'') and '
       .../qa/workunits/ceph-helpers.sh:866: get_num_active_clean:  expression+='not(contains(.,'\''stale'\''))'
       .../qa/workunits/ceph-helpers.sh:867: get_num_active_clean:  expression+=')'
       .../qa/workunits/ceph-helpers.sh:871: get_num_active_clean:  ceph --format xml pg dump pgs
       .../qa/workunits/ceph-helpers.sh:872: get_num_active_clean:  xmlstarlet sel -t -m '//pg_stat/state[(contains(.,'\''active'\'') and contains(.,'\''clean'\'') and not(contains(.,'\''stale'\'')))]' -v . -n
       .../qa/workunits/ceph-helpers.sh:873: get_num_active_clean:  wc -l
       .../qa/workunits/ceph-helpers.sh:873: get_num_active_clean:  grep -v '^$'
       ../qa/workunits/ceph-helpers.sh:993: wait_for_clean:  cur_active_clean=0
       ../qa/workunits/ceph-helpers.sh:994: wait_for_clean:  test 0 '!=' 0
       ../qa/workunits/ceph-helpers.sh:998: wait_for_clean:  sleep 1
       .../qa/workunits/ceph-helpers.sh:999: wait_for_clean:  expr 94 + 1
       ../qa/workunits/ceph-helpers.sh:999: wait_for_clean:  timer=95
       ../qa/workunits/ceph-helpers.sh:985: wait_for_clean:  is_clean
       .../qa/workunits/ceph-helpers.sh:952: is_clean:  get_num_pgs
       .../qa/workunits/ceph-helpers.sh:898: get_num_pgs:  ceph --format xml status
       .../qa/workunits/ceph-helpers.sh:899: get_num_pgs:  xmlstarlet sel -t -m //pgmap/num_pgs -v .
       ../qa/workunits/ceph-helpers.sh:952: is_clean:  num_pgs=4
       ../qa/workunits/ceph-helpers.sh:953: is_clean:  test 4 '!=' 0
       .../qa/workunits/ceph-helpers.sh:954: is_clean:  get_num_active_clean
       .../qa/workunits/ceph-helpers.sh:863: get_num_active_clean:  local 'expression=('
       .../qa/workunits/ceph-helpers.sh:864: get_num_active_clean:  expression+='contains(.,'\''active'\'') and '
       .../qa/workunits/ceph-helpers.sh:865: get_num_active_clean:  expression+='contains(.,'\''clean'\'') and '
       .../qa/workunits/ceph-helpers.sh:866: get_num_active_clean:  expression+='not(contains(.,'\''stale'\''))'
       .../qa/workunits/ceph-helpers.sh:867: get_num_active_clean:  expression+=')'
       .../qa/workunits/ceph-helpers.sh:871: get_num_active_clean:  ceph --format xml pg dump pgs
       .../qa/workunits/ceph-helpers.sh:872: get_num_active_clean:  xmlstarlet sel -t -m '//pg_stat/state[(contains(.,'\''active'\'') and contains(.,'\''clean'\'') and not(contains(.,'\''stale'\'')))]' -v . -n
       .../qa/workunits/ceph-helpers.sh:873: get_num_active_clean:  grep -v '^$'
       .../qa/workunits/ceph-helpers.sh:873: get_num_active_clean:  wc -l
       ../qa/workunits/ceph-helpers.sh:954: is_clean:  test 0 = 4
       ../qa/workunits/ceph-helpers.sh:954: is_clean:  return 1
       ../qa/workunits/ceph-helpers.sh:986: wait_for_clean:  get_is_making_recovery_progress
       .../qa/workunits/ceph-helpers.sh:839: get_is_making_recovery_progress:  ceph --format xml status
       .../qa/workunits/ceph-helpers.sh:839: get_is_making_recovery_progress:  xmlstarlet sel -t -m //pgmap/recovering_keys_per_sec -v . -o ' ' -t -m //pgmap/recovering_bytes_per_sec -v . -o ' ' -t -m //pgmap/recovering_objects_per_sec -v .
       ../qa/workunits/ceph-helpers.sh:839: get_is_making_recovery_progress:  local progress=
       ../qa/workunits/ceph-helpers.sh:840: get_is_making_recovery_progress:  test -n ''
       ../qa/workunits/ceph-helpers.sh:988: wait_for_clean:  ((  timer >= 120  ))
       .../qa/workunits/ceph-helpers.sh:993: wait_for_clean:  get_num_active_clean
       .../qa/workunits/ceph-helpers.sh:863: get_num_active_clean:  local 'expression=('
       .../qa/workunits/ceph-helpers.sh:864: get_num_active_clean:  expression+='contains(.,'\''active'\'') and '
       .../qa/workunits/ceph-helpers.sh:865: get_num_active_clean:  expression+='contains(.,'\''clean'\'') and '
       .../qa/workunits/ceph-helpers.sh:866: get_num_active_clean:  expression+='not(contains(.,'\''stale'\''))'
       .../qa/workunits/ceph-helpers.sh:867: get_num_active_clean:  expression+=')'
       .../qa/workunits/ceph-helpers.sh:871: get_num_active_clean:  ceph --format xml pg dump pgs
       .../qa/workunits/ceph-helpers.sh:872: get_num_active_clean:  xmlstarlet sel -t -m '//pg_stat/state[(contains(.,'\''active'\'') and contains(.,'\''clean'\'') and not(contains(.,'\''stale'\'')))]' -v . -n
       .../qa/workunits/ceph-helpers.sh:873: get_num_active_clean:  grep -v '^$'
       .../qa/workunits/ceph-helpers.sh:873: get_num_active_clean:  wc -l
       ../qa/workunits/ceph-helpers.sh:993: wait_for_clean:  cur_active_clean=0
       ../qa/workunits/ceph-helpers.sh:994: wait_for_clean:  test 0 '!=' 0
       ../qa/workunits/ceph-helpers.sh:998: wait_for_clean:  sleep 1
       .../qa/workunits/ceph-helpers.sh:999: wait_for_clean:  expr 95 + 1
       ../qa/workunits/ceph-helpers.sh:999: wait_for_clean:  timer=96
       ../qa/workunits/ceph-helpers.sh:985: wait_for_clean:  is_clean
       .../qa/workunits/ceph-helpers.sh:952: is_clean:  get_num_pgs
       .../qa/workunits/ceph-helpers.sh:898: get_num_pgs:  ceph --format xml status
       .../qa/workunits/ceph-helpers.sh:899: get_num_pgs:  xmlstarlet sel -t -m //pgmap/num_pgs -v .
       ../qa/workunits/ceph-helpers.sh:952: is_clean:  num_pgs=4
       ../qa/workunits/ceph-helpers.sh:953: is_clean:  test 4 '!=' 0
       .../qa/workunits/ceph-helpers.sh:954: is_clean:  get_num_active_clean
       .../qa/workunits/ceph-helpers.sh:863: get_num_active_clean:  local 'expression=('
       .../qa/workunits/ceph-helpers.sh:864: get_num_active_clean:  expression+='contains(.,'\''active'\'') and '
       .../qa/workunits/ceph-helpers.sh:865: get_num_active_clean:  expression+='contains(.,'\''clean'\'') and '
       .../qa/workunits/ceph-helpers.sh:866: get_num_active_clean:  expression+='not(contains(.,'\''stale'\''))'
       .../qa/workunits/ceph-helpers.sh:867: get_num_active_clean:  expression+=')'
       .../qa/workunits/ceph-helpers.sh:871: get_num_active_clean:  ceph --format xml pg dump pgs
       .../qa/workunits/ceph-helpers.sh:872: get_num_active_clean:  xmlstarlet sel -t -m '//pg_stat/state[(contains(.,'\''active'\'') and contains(.,'\''clean'\'') and not(contains(.,'\''stale'\'')))]' -v . -n
       .../qa/workunits/ceph-helpers.sh:873: get_num_active_clean:  grep -v '^$'
       .../qa/workunits/ceph-helpers.sh:873: get_num_active_clean:  wc -l
       ../qa/workunits/ceph-helpers.sh:954: is_clean:  test 0 = 4
       ../qa/workunits/ceph-helpers.sh:954: is_clean:  return 1
       ../qa/workunits/ceph-helpers.sh:986: wait_for_clean:  get_is_making_recovery_progress
       .../qa/workunits/ceph-helpers.sh:839: get_is_making_recovery_progress:  ceph --format xml status
       .../qa/workunits/ceph-helpers.sh:839: get_is_making_recovery_progress:  xmlstarlet sel -t -m //pgmap/recovering_keys_per_sec -v . -o ' ' -t -m //pgmap/recovering_bytes_per_sec -v . -o ' ' -t -m //pgmap/recovering_objects_per_sec -v .
       ../qa/workunits/ceph-helpers.sh:839: get_is_making_recovery_progress:  local progress=
       ../qa/workunits/ceph-helpers.sh:840: get_is_making_recovery_progress:  test -n ''
       ../qa/workunits/ceph-helpers.sh:988: wait_for_clean:  ((  timer >= 120  ))
       .../qa/workunits/ceph-helpers.sh:993: wait_for_clean:  get_num_active_clean
       .../qa/workunits/ceph-helpers.sh:863: get_num_active_clean:  local 'expression=('
       .../qa/workunits/ceph-helpers.sh:864: get_num_active_clean:  expression+='contains(.,'\''active'\'') and '
       .../qa/workunits/ceph-helpers.sh:865: get_num_active_clean:  expression+='contains(.,'\''clean'\'') and '
       .../qa/workunits/ceph-helpers.sh:866: get_num_active_clean:  expression+='not(contains(.,'\''stale'\''))'
       .../qa/workunits/ceph-helpers.sh:867: get_num_active_clean:  expression+=')'
       .../qa/workunits/ceph-helpers.sh:871: get_num_active_clean:  ceph --format xml pg dump pgs
       .../qa/workunits/ceph-helpers.sh:872: get_num_active_clean:  xmlstarlet sel -t -m '//pg_stat/state[(contains(.,'\''active'\'') and contains(.,'\''clean'\'') and not(contains(.,'\''stale'\'')))]' -v . -n
       .../qa/workunits/ceph-helpers.sh:873: get_num_active_clean:  grep -v '^$'
       .../qa/workunits/ceph-helpers.sh:873: get_num_active_clean:  wc -l
       ../qa/workunits/ceph-helpers.sh:993: wait_for_clean:  cur_active_clean=0
       ../qa/workunits/ceph-helpers.sh:994: wait_for_clean:  test 0 '!=' 0
       ../qa/workunits/ceph-helpers.sh:998: wait_for_clean:  sleep 1
       .../qa/workunits/ceph-helpers.sh:999: wait_for_clean:  expr 96 + 1
       ../qa/workunits/ceph-helpers.sh:999: wait_for_clean:  timer=97
       ../qa/workunits/ceph-helpers.sh:985: wait_for_clean:  is_clean
       .../qa/workunits/ceph-helpers.sh:952: is_clean:  get_num_pgs
       .../qa/workunits/ceph-helpers.sh:898: get_num_pgs:  ceph --format xml status
       .../qa/workunits/ceph-helpers.sh:899: get_num_pgs:  xmlstarlet sel -t -m //pgmap/num_pgs -v .
       ../qa/workunits/ceph-helpers.sh:952: is_clean:  num_pgs=4
       ../qa/workunits/ceph-helpers.sh:953: is_clean:  test 4 '!=' 0
       .../qa/workunits/ceph-helpers.sh:954: is_clean:  get_num_active_clean
       .../qa/workunits/ceph-helpers.sh:863: get_num_active_clean:  local 'expression=('
       .../qa/workunits/ceph-helpers.sh:864: get_num_active_clean:  expression+='contains(.,'\''active'\'') and '
       .../qa/workunits/ceph-helpers.sh:865: get_num_active_clean:  expression+='contains(.,'\''clean'\'') and '
       .../qa/workunits/ceph-helpers.sh:866: get_num_active_clean:  expression+='not(contains(.,'\''stale'\''))'
       .../qa/workunits/ceph-helpers.sh:867: get_num_active_clean:  expression+=')'
       .../qa/workunits/ceph-helpers.sh:871: get_num_active_clean:  ceph --format xml pg dump pgs
       .../qa/workunits/ceph-helpers.sh:872: get_num_active_clean:  xmlstarlet sel -t -m '//pg_stat/state[(contains(.,'\''active'\'') and contains(.,'\''clean'\'') and not(contains(.,'\''stale'\'')))]' -v . -n
       .../qa/workunits/ceph-helpers.sh:873: get_num_active_clean:  wc -l
       .../qa/workunits/ceph-helpers.sh:873: get_num_active_clean:  grep -v '^$'
       ../qa/workunits/ceph-helpers.sh:954: is_clean:  test 0 = 4
       ../qa/workunits/ceph-helpers.sh:954: is_clean:  return 1
       ../qa/workunits/ceph-helpers.sh:986: wait_for_clean:  get_is_making_recovery_progress
       .../qa/workunits/ceph-helpers.sh:839: get_is_making_recovery_progress:  ceph --format xml status
       .../qa/workunits/ceph-helpers.sh:839: get_is_making_recovery_progress:  xmlstarlet sel -t -m //pgmap/recovering_keys_per_sec -v . -o ' ' -t -m //pgmap/recovering_bytes_per_sec -v . -o ' ' -t -m //pgmap/recovering_objects_per_sec -v .
       ../qa/workunits/ceph-helpers.sh:839: get_is_making_recovery_progress:  local progress=
       ../qa/workunits/ceph-helpers.sh:840: get_is_making_recovery_progress:  test -n ''
       ../qa/workunits/ceph-helpers.sh:988: wait_for_clean:  ((  timer >= 120  ))
       .../qa/workunits/ceph-helpers.sh:993: wait_for_clean:  get_num_active_clean
       .../qa/workunits/ceph-helpers.sh:863: get_num_active_clean:  local 'expression=('
       .../qa/workunits/ceph-helpers.sh:864: get_num_active_clean:  expression+='contains(.,'\''active'\'') and '
       .../qa/workunits/ceph-helpers.sh:865: get_num_active_clean:  expression+='contains(.,'\''clean'\'') and '
       .../qa/workunits/ceph-helpers.sh:866: get_num_active_clean:  expression+='not(contains(.,'\''stale'\''))'
       .../qa/workunits/ceph-helpers.sh:867: get_num_active_clean:  expression+=')'
       .../qa/workunits/ceph-helpers.sh:871: get_num_active_clean:  ceph --format xml pg dump pgs
       .../qa/workunits/ceph-helpers.sh:872: get_num_active_clean:  xmlstarlet sel -t -m '//pg_stat/state[(contains(.,'\''active'\'') and contains(.,'\''clean'\'') and not(contains(.,'\''stale'\'')))]' -v . -n
       .../qa/workunits/ceph-helpers.sh:873: get_num_active_clean:  grep -v '^$'
       .../qa/workunits/ceph-helpers.sh:873: get_num_active_clean:  wc -l
       ../qa/workunits/ceph-helpers.sh:993: wait_for_clean:  cur_active_clean=0
       ../qa/workunits/ceph-helpers.sh:994: wait_for_clean:  test 0 '!=' 0
       ../qa/workunits/ceph-helpers.sh:998: wait_for_clean:  sleep 1
       .../qa/workunits/ceph-helpers.sh:999: wait_for_clean:  expr 97 + 1
       ../qa/workunits/ceph-helpers.sh:999: wait_for_clean:  timer=98
       ../qa/workunits/ceph-helpers.sh:985: wait_for_clean:  is_clean
       .../qa/workunits/ceph-helpers.sh:952: is_clean:  get_num_pgs
       .../qa/workunits/ceph-helpers.sh:898: get_num_pgs:  ceph --format xml status
       .../qa/workunits/ceph-helpers.sh:899: get_num_pgs:  xmlstarlet sel -t -m //pgmap/num_pgs -v .
       ../qa/workunits/ceph-helpers.sh:952: is_clean:  num_pgs=4
       ../qa/workunits/ceph-helpers.sh:953: is_clean:  test 4 '!=' 0
       .../qa/workunits/ceph-helpers.sh:954: is_clean:  get_num_active_clean
       .../qa/workunits/ceph-helpers.sh:863: get_num_active_clean:  local 'expression=('
       .../qa/workunits/ceph-helpers.sh:864: get_num_active_clean:  expression+='contains(.,'\''active'\'') and '
       .../qa/workunits/ceph-helpers.sh:865: get_num_active_clean:  expression+='contains(.,'\''clean'\'') and '
       .../qa/workunits/ceph-helpers.sh:866: get_num_active_clean:  expression+='not(contains(.,'\''stale'\''))'
       .../qa/workunits/ceph-helpers.sh:867: get_num_active_clean:  expression+=')'
       .../qa/workunits/ceph-helpers.sh:871: get_num_active_clean:  ceph --format xml pg dump pgs
       .../qa/workunits/ceph-helpers.sh:872: get_num_active_clean:  xmlstarlet sel -t -m '//pg_stat/state[(contains(.,'\''active'\'') and contains(.,'\''clean'\'') and not(contains(.,'\''stale'\'')))]' -v . -n
       .../qa/workunits/ceph-helpers.sh:873: get_num_active_clean:  grep -v '^$'
       .../qa/workunits/ceph-helpers.sh:873: get_num_active_clean:  wc -l
       ../qa/workunits/ceph-helpers.sh:954: is_clean:  test 0 = 4
       ../qa/workunits/ceph-helpers.sh:954: is_clean:  return 1
       ../qa/workunits/ceph-helpers.sh:986: wait_for_clean:  get_is_making_recovery_progress
       .../qa/workunits/ceph-helpers.sh:839: get_is_making_recovery_progress:  ceph --format xml status
       .../qa/workunits/ceph-helpers.sh:839: get_is_making_recovery_progress:  xmlstarlet sel -t -m //pgmap/recovering_keys_per_sec -v . -o ' ' -t -m //pgmap/recovering_bytes_per_sec -v . -o ' ' -t -m //pgmap/recovering_objects_per_sec -v .
       ../qa/workunits/ceph-helpers.sh:839: get_is_making_recovery_progress:  local progress=
       ../qa/workunits/ceph-helpers.sh:840: get_is_making_recovery_progress:  test -n ''
       ../qa/workunits/ceph-helpers.sh:988: wait_for_clean:  ((  timer >= 120  ))
       .../qa/workunits/ceph-helpers.sh:993: wait_for_clean:  get_num_active_clean
       .../qa/workunits/ceph-helpers.sh:863: get_num_active_clean:  local 'expression=('
       .../qa/workunits/ceph-helpers.sh:864: get_num_active_clean:  expression+='contains(.,'\''active'\'') and '
       .../qa/workunits/ceph-helpers.sh:865: get_num_active_clean:  expression+='contains(.,'\''clean'\'') and '
       .../qa/workunits/ceph-helpers.sh:866: get_num_active_clean:  expression+='not(contains(.,'\''stale'\''))'
       .../qa/workunits/ceph-helpers.sh:867: get_num_active_clean:  expression+=')'
       .../qa/workunits/ceph-helpers.sh:871: get_num_active_clean:  ceph --format xml pg dump pgs
       .../qa/workunits/ceph-helpers.sh:872: get_num_active_clean:  xmlstarlet sel -t -m '//pg_stat/state[(contains(.,'\''active'\'') and contains(.,'\''clean'\'') and not(contains(.,'\''stale'\'')))]' -v . -n
       .../qa/workunits/ceph-helpers.sh:873: get_num_active_clean:  grep -v '^$'
       .../qa/workunits/ceph-helpers.sh:873: get_num_active_clean:  wc -l
       ../qa/workunits/ceph-helpers.sh:993: wait_for_clean:  cur_active_clean=0
       ../qa/workunits/ceph-helpers.sh:994: wait_for_clean:  test 0 '!=' 0
       ../qa/workunits/ceph-helpers.sh:998: wait_for_clean:  sleep 1
       .../qa/workunits/ceph-helpers.sh:999: wait_for_clean:  expr 98 + 1
       ../qa/workunits/ceph-helpers.sh:999: wait_for_clean:  timer=99
       ../qa/workunits/ceph-helpers.sh:985: wait_for_clean:  is_clean
       .../qa/workunits/ceph-helpers.sh:952: is_clean:  get_num_pgs
       .../qa/workunits/ceph-helpers.sh:898: get_num_pgs:  ceph --format xml status
       .../qa/workunits/ceph-helpers.sh:899: get_num_pgs:  xmlstarlet sel -t -m //pgmap/num_pgs -v .
       ../qa/workunits/ceph-helpers.sh:952: is_clean:  num_pgs=4
       ../qa/workunits/ceph-helpers.sh:953: is_clean:  test 4 '!=' 0
       .../qa/workunits/ceph-helpers.sh:954: is_clean:  get_num_active_clean
       .../qa/workunits/ceph-helpers.sh:863: get_num_active_clean:  local 'expression=('
       .../qa/workunits/ceph-helpers.sh:864: get_num_active_clean:  expression+='contains(.,'\''active'\'') and '
       .../qa/workunits/ceph-helpers.sh:865: get_num_active_clean:  expression+='contains(.,'\''clean'\'') and '
       .../qa/workunits/ceph-helpers.sh:866: get_num_active_clean:  expression+='not(contains(.,'\''stale'\''))'
       .../qa/workunits/ceph-helpers.sh:867: get_num_active_clean:  expression+=')'
       .../qa/workunits/ceph-helpers.sh:871: get_num_active_clean:  ceph --format xml pg dump pgs
       .../qa/workunits/ceph-helpers.sh:872: get_num_active_clean:  xmlstarlet sel -t -m '//pg_stat/state[(contains(.,'\''active'\'') and contains(.,'\''clean'\'') and not(contains(.,'\''stale'\'')))]' -v . -n
       .../qa/workunits/ceph-helpers.sh:873: get_num_active_clean:  grep -v '^$'
       .../qa/workunits/ceph-helpers.sh:873: get_num_active_clean:  wc -l
       ../qa/workunits/ceph-helpers.sh:954: is_clean:  test 0 = 4
       ../qa/workunits/ceph-helpers.sh:954: is_clean:  return 1
       ../qa/workunits/ceph-helpers.sh:986: wait_for_clean:  get_is_making_recovery_progress
       .../qa/workunits/ceph-helpers.sh:839: get_is_making_recovery_progress:  ceph --format xml status
       .../qa/workunits/ceph-helpers.sh:839: get_is_making_recovery_progress:  xmlstarlet sel -t -m //pgmap/recovering_keys_per_sec -v . -o ' ' -t -m //pgmap/recovering_bytes_per_sec -v . -o ' ' -t -m //pgmap/recovering_objects_per_sec -v .
       ../qa/workunits/ceph-helpers.sh:839: get_is_making_recovery_progress:  local progress=
       ../qa/workunits/ceph-helpers.sh:840: get_is_making_recovery_progress:  test -n ''
       ../qa/workunits/ceph-helpers.sh:988: wait_for_clean:  ((  timer >= 120  ))
       .../qa/workunits/ceph-helpers.sh:993: wait_for_clean:  get_num_active_clean
       .../qa/workunits/ceph-helpers.sh:863: get_num_active_clean:  local 'expression=('
       .../qa/workunits/ceph-helpers.sh:864: get_num_active_clean:  expression+='contains(.,'\''active'\'') and '
       .../qa/workunits/ceph-helpers.sh:865: get_num_active_clean:  expression+='contains(.,'\''clean'\'') and '
       .../qa/workunits/ceph-helpers.sh:866: get_num_active_clean:  expression+='not(contains(.,'\''stale'\''))'
       .../qa/workunits/ceph-helpers.sh:867: get_num_active_clean:  expression+=')'
       .../qa/workunits/ceph-helpers.sh:871: get_num_active_clean:  ceph --format xml pg dump pgs
       .../qa/workunits/ceph-helpers.sh:872: get_num_active_clean:  xmlstarlet sel -t -m '//pg_stat/state[(contains(.,'\''active'\'') and contains(.,'\''clean'\'') and not(contains(.,'\''stale'\'')))]' -v . -n
       .../qa/workunits/ceph-helpers.sh:873: get_num_active_clean:  grep -v '^$'
       .../qa/workunits/ceph-helpers.sh:873: get_num_active_clean:  wc -l
       ../qa/workunits/ceph-helpers.sh:993: wait_for_clean:  cur_active_clean=0
       ../qa/workunits/ceph-helpers.sh:994: wait_for_clean:  test 0 '!=' 0
       ../qa/workunits/ceph-helpers.sh:998: wait_for_clean:  sleep 1
       .../qa/workunits/ceph-helpers.sh:999: wait_for_clean:  expr 99 + 1
       ../qa/workunits/ceph-helpers.sh:999: wait_for_clean:  timer=100
       ../qa/workunits/ceph-helpers.sh:985: wait_for_clean:  is_clean
       .../qa/workunits/ceph-helpers.sh:952: is_clean:  get_num_pgs
       .../qa/workunits/ceph-helpers.sh:898: get_num_pgs:  ceph --format xml status
       .../qa/workunits/ceph-helpers.sh:899: get_num_pgs:  xmlstarlet sel -t -m //pgmap/num_pgs -v .
       ../qa/workunits/ceph-helpers.sh:952: is_clean:  num_pgs=4
       ../qa/workunits/ceph-helpers.sh:953: is_clean:  test 4 '!=' 0
       .../qa/workunits/ceph-helpers.sh:954: is_clean:  get_num_active_clean
       .../qa/workunits/ceph-helpers.sh:863: get_num_active_clean:  local 'expression=('
       .../qa/workunits/ceph-helpers.sh:864: get_num_active_clean:  expression+='contains(.,'\''active'\'') and '
       .../qa/workunits/ceph-helpers.sh:865: get_num_active_clean:  expression+='contains(.,'\''clean'\'') and '
       .../qa/workunits/ceph-helpers.sh:866: get_num_active_clean:  expression+='not(contains(.,'\''stale'\''))'
       .../qa/workunits/ceph-helpers.sh:867: get_num_active_clean:  expression+=')'
       .../qa/workunits/ceph-helpers.sh:871: get_num_active_clean:  ceph --format xml pg dump pgs
       .../qa/workunits/ceph-helpers.sh:872: get_num_active_clean:  xmlstarlet sel -t -m '//pg_stat/state[(contains(.,'\''active'\'') and contains(.,'\''clean'\'') and not(contains(.,'\''stale'\'')))]' -v . -n
       .../qa/workunits/ceph-helpers.sh:873: get_num_active_clean:  wc -l
       .../qa/workunits/ceph-helpers.sh:873: get_num_active_clean:  grep -v '^$'
       ../qa/workunits/ceph-helpers.sh:954: is_clean:  test 0 = 4
       ../qa/workunits/ceph-helpers.sh:954: is_clean:  return 1
       ../qa/workunits/ceph-helpers.sh:986: wait_for_clean:  get_is_making_recovery_progress
       .../qa/workunits/ceph-helpers.sh:839: get_is_making_recovery_progress:  ceph --format xml status
       .../qa/workunits/ceph-helpers.sh:839: get_is_making_recovery_progress:  xmlstarlet sel -t -m //pgmap/recovering_keys_per_sec -v . -o ' ' -t -m //pgmap/recovering_bytes_per_sec -v . -o ' ' -t -m //pgmap/recovering_objects_per_sec -v .
       ../qa/workunits/ceph-helpers.sh:839: get_is_making_recovery_progress:  local progress=
       ../qa/workunits/ceph-helpers.sh:840: get_is_making_recovery_progress:  test -n ''
       ../qa/workunits/ceph-helpers.sh:988: wait_for_clean:  ((  timer >= 120  ))
       .../qa/workunits/ceph-helpers.sh:993: wait_for_clean:  get_num_active_clean
       .../qa/workunits/ceph-helpers.sh:863: get_num_active_clean:  local 'expression=('
       .../qa/workunits/ceph-helpers.sh:864: get_num_active_clean:  expression+='contains(.,'\''active'\'') and '
       .../qa/workunits/ceph-helpers.sh:865: get_num_active_clean:  expression+='contains(.,'\''clean'\'') and '
       .../qa/workunits/ceph-helpers.sh:866: get_num_active_clean:  expression+='not(contains(.,'\''stale'\''))'
       .../qa/workunits/ceph-helpers.sh:867: get_num_active_clean:  expression+=')'
       .../qa/workunits/ceph-helpers.sh:871: get_num_active_clean:  ceph --format xml pg dump pgs
       .../qa/workunits/ceph-helpers.sh:872: get_num_active_clean:  xmlstarlet sel -t -m '//pg_stat/state[(contains(.,'\''active'\'') and contains(.,'\''clean'\'') and not(contains(.,'\''stale'\'')))]' -v . -n
       .../qa/workunits/ceph-helpers.sh:873: get_num_active_clean:  wc -l
       .../qa/workunits/ceph-helpers.sh:873: get_num_active_clean:  grep -v '^$'
       ../qa/workunits/ceph-helpers.sh:993: wait_for_clean:  cur_active_clean=0
       ../qa/workunits/ceph-helpers.sh:994: wait_for_clean:  test 0 '!=' 0
       ../qa/workunits/ceph-helpers.sh:998: wait_for_clean:  sleep 1
       .../qa/workunits/ceph-helpers.sh:999: wait_for_clean:  expr 100 + 1
       ../qa/workunits/ceph-helpers.sh:999: wait_for_clean:  timer=101
       ../qa/workunits/ceph-helpers.sh:985: wait_for_clean:  is_clean
       .../qa/workunits/ceph-helpers.sh:952: is_clean:  get_num_pgs
       .../qa/workunits/ceph-helpers.sh:898: get_num_pgs:  ceph --format xml status
       .../qa/workunits/ceph-helpers.sh:899: get_num_pgs:  xmlstarlet sel -t -m //pgmap/num_pgs -v .
       ../qa/workunits/ceph-helpers.sh:952: is_clean:  num_pgs=4
       ../qa/workunits/ceph-helpers.sh:953: is_clean:  test 4 '!=' 0
       .../qa/workunits/ceph-helpers.sh:954: is_clean:  get_num_active_clean
       .../qa/workunits/ceph-helpers.sh:863: get_num_active_clean:  local 'expression=('
       .../qa/workunits/ceph-helpers.sh:864: get_num_active_clean:  expression+='contains(.,'\''active'\'') and '
       .../qa/workunits/ceph-helpers.sh:865: get_num_active_clean:  expression+='contains(.,'\''clean'\'') and '
       .../qa/workunits/ceph-helpers.sh:866: get_num_active_clean:  expression+='not(contains(.,'\''stale'\''))'
       .../qa/workunits/ceph-helpers.sh:867: get_num_active_clean:  expression+=')'
       .../qa/workunits/ceph-helpers.sh:871: get_num_active_clean:  ceph --format xml pg dump pgs
       .../qa/workunits/ceph-helpers.sh:872: get_num_active_clean:  xmlstarlet sel -t -m '//pg_stat/state[(contains(.,'\''active'\'') and contains(.,'\''clean'\'') and not(contains(.,'\''stale'\'')))]' -v . -n
       .../qa/workunits/ceph-helpers.sh:873: get_num_active_clean:  grep -v '^$'
       .../qa/workunits/ceph-helpers.sh:873: get_num_active_clean:  wc -l
       ../qa/workunits/ceph-helpers.sh:954: is_clean:  test 0 = 4
       ../qa/workunits/ceph-helpers.sh:954: is_clean:  return 1
       ../qa/workunits/ceph-helpers.sh:986: wait_for_clean:  get_is_making_recovery_progress
       .../qa/workunits/ceph-helpers.sh:839: get_is_making_recovery_progress:  ceph --format xml status
       .../qa/workunits/ceph-helpers.sh:839: get_is_making_recovery_progress:  xmlstarlet sel -t -m //pgmap/recovering_keys_per_sec -v . -o ' ' -t -m //pgmap/recovering_bytes_per_sec -v . -o ' ' -t -m //pgmap/recovering_objects_per_sec -v .
       ../qa/workunits/ceph-helpers.sh:839: get_is_making_recovery_progress:  local progress=
       ../qa/workunits/ceph-helpers.sh:840: get_is_making_recovery_progress:  test -n ''
       ../qa/workunits/ceph-helpers.sh:988: wait_for_clean:  ((  timer >= 120  ))
       .../qa/workunits/ceph-helpers.sh:993: wait_for_clean:  get_num_active_clean
       .../qa/workunits/ceph-helpers.sh:863: get_num_active_clean:  local 'expression=('
       .../qa/workunits/ceph-helpers.sh:864: get_num_active_clean:  expression+='contains(.,'\''active'\'') and '
       .../qa/workunits/ceph-helpers.sh:865: get_num_active_clean:  expression+='contains(.,'\''clean'\'') and '
       .../qa/workunits/ceph-helpers.sh:866: get_num_active_clean:  expression+='not(contains(.,'\''stale'\''))'
       .../qa/workunits/ceph-helpers.sh:867: get_num_active_clean:  expression+=')'
       .../qa/workunits/ceph-helpers.sh:871: get_num_active_clean:  ceph --format xml pg dump pgs
       .../qa/workunits/ceph-helpers.sh:872: get_num_active_clean:  xmlstarlet sel -t -m '//pg_stat/state[(contains(.,'\''active'\'') and contains(.,'\''clean'\'') and not(contains(.,'\''stale'\'')))]' -v . -n
       .../qa/workunits/ceph-helpers.sh:873: get_num_active_clean:  grep -v '^$'
       .../qa/workunits/ceph-helpers.sh:873: get_num_active_clean:  wc -l
       ../qa/workunits/ceph-helpers.sh:993: wait_for_clean:  cur_active_clean=0
       ../qa/workunits/ceph-helpers.sh:994: wait_for_clean:  test 0 '!=' 0
       ../qa/workunits/ceph-helpers.sh:998: wait_for_clean:  sleep 1
       .../qa/workunits/ceph-helpers.sh:999: wait_for_clean:  expr 101 + 1
       ../qa/workunits/ceph-helpers.sh:999: wait_for_clean:  timer=102
       ../qa/workunits/ceph-helpers.sh:985: wait_for_clean:  is_clean
       .../qa/workunits/ceph-helpers.sh:952: is_clean:  get_num_pgs
       .../qa/workunits/ceph-helpers.sh:898: get_num_pgs:  ceph --format xml status
       .../qa/workunits/ceph-helpers.sh:899: get_num_pgs:  xmlstarlet sel -t -m //pgmap/num_pgs -v .
       ../qa/workunits/ceph-helpers.sh:952: is_clean:  num_pgs=4
       ../qa/workunits/ceph-helpers.sh:953: is_clean:  test 4 '!=' 0
       .../qa/workunits/ceph-helpers.sh:954: is_clean:  get_num_active_clean
       .../qa/workunits/ceph-helpers.sh:863: get_num_active_clean:  local 'expression=('
       .../qa/workunits/ceph-helpers.sh:864: get_num_active_clean:  expression+='contains(.,'\''active'\'') and '
       .../qa/workunits/ceph-helpers.sh:865: get_num_active_clean:  expression+='contains(.,'\''clean'\'') and '
       .../qa/workunits/ceph-helpers.sh:866: get_num_active_clean:  expression+='not(contains(.,'\''stale'\''))'
       .../qa/workunits/ceph-helpers.sh:867: get_num_active_clean:  expression+=')'
       .../qa/workunits/ceph-helpers.sh:871: get_num_active_clean:  ceph --format xml pg dump pgs
       .../qa/workunits/ceph-helpers.sh:872: get_num_active_clean:  xmlstarlet sel -t -m '//pg_stat/state[(contains(.,'\''active'\'') and contains(.,'\''clean'\'') and not(contains(.,'\''stale'\'')))]' -v . -n
       .../qa/workunits/ceph-helpers.sh:873: get_num_active_clean:  grep -v '^$'
       .../qa/workunits/ceph-helpers.sh:873: get_num_active_clean:  wc -l
       ../qa/workunits/ceph-helpers.sh:954: is_clean:  test 0 = 4
       ../qa/workunits/ceph-helpers.sh:954: is_clean:  return 1
       ../qa/workunits/ceph-helpers.sh:986: wait_for_clean:  get_is_making_recovery_progress
       .../qa/workunits/ceph-helpers.sh:839: get_is_making_recovery_progress:  ceph --format xml status
       .../qa/workunits/ceph-helpers.sh:839: get_is_making_recovery_progress:  xmlstarlet sel -t -m //pgmap/recovering_keys_per_sec -v . -o ' ' -t -m //pgmap/recovering_bytes_per_sec -v . -o ' ' -t -m //pgmap/recovering_objects_per_sec -v .
       ../qa/workunits/ceph-helpers.sh:839: get_is_making_recovery_progress:  local progress=
       ../qa/workunits/ceph-helpers.sh:840: get_is_making_recovery_progress:  test -n ''
       ../qa/workunits/ceph-helpers.sh:988: wait_for_clean:  ((  timer >= 120  ))
       .../qa/workunits/ceph-helpers.sh:993: wait_for_clean:  get_num_active_clean
       .../qa/workunits/ceph-helpers.sh:863: get_num_active_clean:  local 'expression=('
       .../qa/workunits/ceph-helpers.sh:864: get_num_active_clean:  expression+='contains(.,'\''active'\'') and '
       .../qa/workunits/ceph-helpers.sh:865: get_num_active_clean:  expression+='contains(.,'\''clean'\'') and '
       .../qa/workunits/ceph-helpers.sh:866: get_num_active_clean:  expression+='not(contains(.,'\''stale'\''))'
       .../qa/workunits/ceph-helpers.sh:867: get_num_active_clean:  expression+=')'
       .../qa/workunits/ceph-helpers.sh:871: get_num_active_clean:  ceph --format xml pg dump pgs
       .../qa/workunits/ceph-helpers.sh:872: get_num_active_clean:  xmlstarlet sel -t -m '//pg_stat/state[(contains(.,'\''active'\'') and contains(.,'\''clean'\'') and not(contains(.,'\''stale'\'')))]' -v . -n
       .../qa/workunits/ceph-helpers.sh:873: get_num_active_clean:  grep -v '^$'
       .../qa/workunits/ceph-helpers.sh:873: get_num_active_clean:  wc -l
       ../qa/workunits/ceph-helpers.sh:993: wait_for_clean:  cur_active_clean=0
       ../qa/workunits/ceph-helpers.sh:994: wait_for_clean:  test 0 '!=' 0
       ../qa/workunits/ceph-helpers.sh:998: wait_for_clean:  sleep 1
       .../qa/workunits/ceph-helpers.sh:999: wait_for_clean:  expr 102 + 1
       ../qa/workunits/ceph-helpers.sh:999: wait_for_clean:  timer=103
       ../qa/workunits/ceph-helpers.sh:985: wait_for_clean:  is_clean
       .../qa/workunits/ceph-helpers.sh:952: is_clean:  get_num_pgs
       .../qa/workunits/ceph-helpers.sh:898: get_num_pgs:  ceph --format xml status
       .../qa/workunits/ceph-helpers.sh:899: get_num_pgs:  xmlstarlet sel -t -m //pgmap/num_pgs -v .
       ../qa/workunits/ceph-helpers.sh:952: is_clean:  num_pgs=4
       ../qa/workunits/ceph-helpers.sh:953: is_clean:  test 4 '!=' 0
       .../qa/workunits/ceph-helpers.sh:954: is_clean:  get_num_active_clean
       .../qa/workunits/ceph-helpers.sh:863: get_num_active_clean:  local 'expression=('
       .../qa/workunits/ceph-helpers.sh:864: get_num_active_clean:  expression+='contains(.,'\''active'\'') and '
       .../qa/workunits/ceph-helpers.sh:865: get_num_active_clean:  expression+='contains(.,'\''clean'\'') and '
       .../qa/workunits/ceph-helpers.sh:866: get_num_active_clean:  expression+='not(contains(.,'\''stale'\''))'
       .../qa/workunits/ceph-helpers.sh:867: get_num_active_clean:  expression+=')'
       .../qa/workunits/ceph-helpers.sh:871: get_num_active_clean:  ceph --format xml pg dump pgs
       .../qa/workunits/ceph-helpers.sh:872: get_num_active_clean:  xmlstarlet sel -t -m '//pg_stat/state[(contains(.,'\''active'\'') and contains(.,'\''clean'\'') and not(contains(.,'\''stale'\'')))]' -v . -n
       .../qa/workunits/ceph-helpers.sh:873: get_num_active_clean:  grep -v '^$'
       .../qa/workunits/ceph-helpers.sh:873: get_num_active_clean:  wc -l
       ../qa/workunits/ceph-helpers.sh:954: is_clean:  test 0 = 4
       ../qa/workunits/ceph-helpers.sh:954: is_clean:  return 1
       ../qa/workunits/ceph-helpers.sh:986: wait_for_clean:  get_is_making_recovery_progress
       .../qa/workunits/ceph-helpers.sh:839: get_is_making_recovery_progress:  ceph --format xml status
       .../qa/workunits/ceph-helpers.sh:839: get_is_making_recovery_progress:  xmlstarlet sel -t -m //pgmap/recovering_keys_per_sec -v . -o ' ' -t -m //pgmap/recovering_bytes_per_sec -v . -o ' ' -t -m //pgmap/recovering_objects_per_sec -v .
       ../qa/workunits/ceph-helpers.sh:839: get_is_making_recovery_progress:  local progress=
       ../qa/workunits/ceph-helpers.sh:840: get_is_making_recovery_progress:  test -n ''
       ../qa/workunits/ceph-helpers.sh:988: wait_for_clean:  ((  timer >= 120  ))
       .../qa/workunits/ceph-helpers.sh:993: wait_for_clean:  get_num_active_clean
       .../qa/workunits/ceph-helpers.sh:863: get_num_active_clean:  local 'expression=('
       .../qa/workunits/ceph-helpers.sh:864: get_num_active_clean:  expression+='contains(.,'\''active'\'') and '
       .../qa/workunits/ceph-helpers.sh:865: get_num_active_clean:  expression+='contains(.,'\''clean'\'') and '
       .../qa/workunits/ceph-helpers.sh:866: get_num_active_clean:  expression+='not(contains(.,'\''stale'\''))'
       .../qa/workunits/ceph-helpers.sh:867: get_num_active_clean:  expression+=')'
       .../qa/workunits/ceph-helpers.sh:871: get_num_active_clean:  ceph --format xml pg dump pgs
       .../qa/workunits/ceph-helpers.sh:872: get_num_active_clean:  xmlstarlet sel -t -m '//pg_stat/state[(contains(.,'\''active'\'') and contains(.,'\''clean'\'') and not(contains(.,'\''stale'\'')))]' -v . -n
       .../qa/workunits/ceph-helpers.sh:873: get_num_active_clean:  grep -v '^$'
       .../qa/workunits/ceph-helpers.sh:873: get_num_active_clean:  wc -l
       ../qa/workunits/ceph-helpers.sh:993: wait_for_clean:  cur_active_clean=0
       ../qa/workunits/ceph-helpers.sh:994: wait_for_clean:  test 0 '!=' 0
       ../qa/workunits/ceph-helpers.sh:998: wait_for_clean:  sleep 1
       .../qa/workunits/ceph-helpers.sh:999: wait_for_clean:  expr 103 + 1
       ../qa/workunits/ceph-helpers.sh:999: wait_for_clean:  timer=104
       ../qa/workunits/ceph-helpers.sh:985: wait_for_clean:  is_clean
       .../qa/workunits/ceph-helpers.sh:952: is_clean:  get_num_pgs
       .../qa/workunits/ceph-helpers.sh:898: get_num_pgs:  ceph --format xml status
       .../qa/workunits/ceph-helpers.sh:899: get_num_pgs:  xmlstarlet sel -t -m //pgmap/num_pgs -v .
       ../qa/workunits/ceph-helpers.sh:952: is_clean:  num_pgs=4
       ../qa/workunits/ceph-helpers.sh:953: is_clean:  test 4 '!=' 0
       .../qa/workunits/ceph-helpers.sh:954: is_clean:  get_num_active_clean
       .../qa/workunits/ceph-helpers.sh:863: get_num_active_clean:  local 'expression=('
       .../qa/workunits/ceph-helpers.sh:864: get_num_active_clean:  expression+='contains(.,'\''active'\'') and '
       .../qa/workunits/ceph-helpers.sh:865: get_num_active_clean:  expression+='contains(.,'\''clean'\'') and '
       .../qa/workunits/ceph-helpers.sh:866: get_num_active_clean:  expression+='not(contains(.,'\''stale'\''))'
       .../qa/workunits/ceph-helpers.sh:867: get_num_active_clean:  expression+=')'
       .../qa/workunits/ceph-helpers.sh:871: get_num_active_clean:  ceph --format xml pg dump pgs
       .../qa/workunits/ceph-helpers.sh:872: get_num_active_clean:  xmlstarlet sel -t -m '//pg_stat/state[(contains(.,'\''active'\'') and contains(.,'\''clean'\'') and not(contains(.,'\''stale'\'')))]' -v . -n
       .../qa/workunits/ceph-helpers.sh:873: get_num_active_clean:  wc -l
       .../qa/workunits/ceph-helpers.sh:873: get_num_active_clean:  grep -v '^$'
       ../qa/workunits/ceph-helpers.sh:954: is_clean:  test 0 = 4
       ../qa/workunits/ceph-helpers.sh:954: is_clean:  return 1
       ../qa/workunits/ceph-helpers.sh:986: wait_for_clean:  get_is_making_recovery_progress
       .../qa/workunits/ceph-helpers.sh:839: get_is_making_recovery_progress:  ceph --format xml status
       .../qa/workunits/ceph-helpers.sh:839: get_is_making_recovery_progress:  xmlstarlet sel -t -m //pgmap/recovering_keys_per_sec -v . -o ' ' -t -m //pgmap/recovering_bytes_per_sec -v . -o ' ' -t -m //pgmap/recovering_objects_per_sec -v .
       ../qa/workunits/ceph-helpers.sh:839: get_is_making_recovery_progress:  local progress=
       ../qa/workunits/ceph-helpers.sh:840: get_is_making_recovery_progress:  test -n ''
       ../qa/workunits/ceph-helpers.sh:988: wait_for_clean:  ((  timer >= 120  ))
       .../qa/workunits/ceph-helpers.sh:993: wait_for_clean:  get_num_active_clean
       .../qa/workunits/ceph-helpers.sh:863: get_num_active_clean:  local 'expression=('
       .../qa/workunits/ceph-helpers.sh:864: get_num_active_clean:  expression+='contains(.,'\''active'\'') and '
       .../qa/workunits/ceph-helpers.sh:865: get_num_active_clean:  expression+='contains(.,'\''clean'\'') and '
       .../qa/workunits/ceph-helpers.sh:866: get_num_active_clean:  expression+='not(contains(.,'\''stale'\''))'
       .../qa/workunits/ceph-helpers.sh:867: get_num_active_clean:  expression+=')'
       .../qa/workunits/ceph-helpers.sh:871: get_num_active_clean:  ceph --format xml pg dump pgs
       .../qa/workunits/ceph-helpers.sh:872: get_num_active_clean:  xmlstarlet sel -t -m '//pg_stat/state[(contains(.,'\''active'\'') and contains(.,'\''clean'\'') and not(contains(.,'\''stale'\'')))]' -v . -n
       .../qa/workunits/ceph-helpers.sh:873: get_num_active_clean:  wc -l
       .../qa/workunits/ceph-helpers.sh:873: get_num_active_clean:  grep -v '^$'
       ../qa/workunits/ceph-helpers.sh:993: wait_for_clean:  cur_active_clean=0
       ../qa/workunits/ceph-helpers.sh:994: wait_for_clean:  test 0 '!=' 0
       ../qa/workunits/ceph-helpers.sh:998: wait_for_clean:  sleep 1
       .../qa/workunits/ceph-helpers.sh:999: wait_for_clean:  expr 104 + 1
       ../qa/workunits/ceph-helpers.sh:999: wait_for_clean:  timer=105
       ../qa/workunits/ceph-helpers.sh:985: wait_for_clean:  is_clean
       .../qa/workunits/ceph-helpers.sh:952: is_clean:  get_num_pgs
       .../qa/workunits/ceph-helpers.sh:898: get_num_pgs:  ceph --format xml status
       .../qa/workunits/ceph-helpers.sh:899: get_num_pgs:  xmlstarlet sel -t -m //pgmap/num_pgs -v .
       ../qa/workunits/ceph-helpers.sh:952: is_clean:  num_pgs=4
       ../qa/workunits/ceph-helpers.sh:953: is_clean:  test 4 '!=' 0
       .../qa/workunits/ceph-helpers.sh:954: is_clean:  get_num_active_clean
       .../qa/workunits/ceph-helpers.sh:863: get_num_active_clean:  local 'expression=('
       .../qa/workunits/ceph-helpers.sh:864: get_num_active_clean:  expression+='contains(.,'\''active'\'') and '
       .../qa/workunits/ceph-helpers.sh:865: get_num_active_clean:  expression+='contains(.,'\''clean'\'') and '
       .../qa/workunits/ceph-helpers.sh:866: get_num_active_clean:  expression+='not(contains(.,'\''stale'\''))'
       .../qa/workunits/ceph-helpers.sh:867: get_num_active_clean:  expression+=')'
       .../qa/workunits/ceph-helpers.sh:871: get_num_active_clean:  ceph --format xml pg dump pgs
       .../qa/workunits/ceph-helpers.sh:872: get_num_active_clean:  xmlstarlet sel -t -m '//pg_stat/state[(contains(.,'\''active'\'') and contains(.,'\''clean'\'') and not(contains(.,'\''stale'\'')))]' -v . -n
       .../qa/workunits/ceph-helpers.sh:873: get_num_active_clean:  grep -v '^$'
       .../qa/workunits/ceph-helpers.sh:873: get_num_active_clean:  wc -l
       ../qa/workunits/ceph-helpers.sh:954: is_clean:  test 0 = 4
       ../qa/workunits/ceph-helpers.sh:954: is_clean:  return 1
       ../qa/workunits/ceph-helpers.sh:986: wait_for_clean:  get_is_making_recovery_progress
       .../qa/workunits/ceph-helpers.sh:839: get_is_making_recovery_progress:  ceph --format xml status
       .../qa/workunits/ceph-helpers.sh:839: get_is_making_recovery_progress:  xmlstarlet sel -t -m //pgmap/recovering_keys_per_sec -v . -o ' ' -t -m //pgmap/recovering_bytes_per_sec -v . -o ' ' -t -m //pgmap/recovering_objects_per_sec -v .
       ../qa/workunits/ceph-helpers.sh:839: get_is_making_recovery_progress:  local progress=
       ../qa/workunits/ceph-helpers.sh:840: get_is_making_recovery_progress:  test -n ''
       ../qa/workunits/ceph-helpers.sh:988: wait_for_clean:  ((  timer >= 120  ))
       .../qa/workunits/ceph-helpers.sh:993: wait_for_clean:  get_num_active_clean
       .../qa/workunits/ceph-helpers.sh:863: get_num_active_clean:  local 'expression=('
       .../qa/workunits/ceph-helpers.sh:864: get_num_active_clean:  expression+='contains(.,'\''active'\'') and '
       .../qa/workunits/ceph-helpers.sh:865: get_num_active_clean:  expression+='contains(.,'\''clean'\'') and '
       .../qa/workunits/ceph-helpers.sh:866: get_num_active_clean:  expression+='not(contains(.,'\''stale'\''))'
       .../qa/workunits/ceph-helpers.sh:867: get_num_active_clean:  expression+=')'
       .../qa/workunits/ceph-helpers.sh:871: get_num_active_clean:  ceph --format xml pg dump pgs
       .../qa/workunits/ceph-helpers.sh:872: get_num_active_clean:  xmlstarlet sel -t -m '//pg_stat/state[(contains(.,'\''active'\'') and contains(.,'\''clean'\'') and not(contains(.,'\''stale'\'')))]' -v . -n
       .../qa/workunits/ceph-helpers.sh:873: get_num_active_clean:  grep -v '^$'
       .../qa/workunits/ceph-helpers.sh:873: get_num_active_clean:  wc -l
       ../qa/workunits/ceph-helpers.sh:993: wait_for_clean:  cur_active_clean=0
       ../qa/workunits/ceph-helpers.sh:994: wait_for_clean:  test 0 '!=' 0
       ../qa/workunits/ceph-helpers.sh:998: wait_for_clean:  sleep 1
       .../qa/workunits/ceph-helpers.sh:999: wait_for_clean:  expr 105 + 1
       ../qa/workunits/ceph-helpers.sh:999: wait_for_clean:  timer=106
       ../qa/workunits/ceph-helpers.sh:985: wait_for_clean:  is_clean
       .../qa/workunits/ceph-helpers.sh:952: is_clean:  get_num_pgs
       .../qa/workunits/ceph-helpers.sh:898: get_num_pgs:  ceph --format xml status
       .../qa/workunits/ceph-helpers.sh:899: get_num_pgs:  xmlstarlet sel -t -m //pgmap/num_pgs -v .
       ../qa/workunits/ceph-helpers.sh:952: is_clean:  num_pgs=4
       ../qa/workunits/ceph-helpers.sh:953: is_clean:  test 4 '!=' 0
       .../qa/workunits/ceph-helpers.sh:954: is_clean:  get_num_active_clean
       .../qa/workunits/ceph-helpers.sh:863: get_num_active_clean:  local 'expression=('
       .../qa/workunits/ceph-helpers.sh:864: get_num_active_clean:  expression+='contains(.,'\''active'\'') and '
       .../qa/workunits/ceph-helpers.sh:865: get_num_active_clean:  expression+='contains(.,'\''clean'\'') and '
       .../qa/workunits/ceph-helpers.sh:866: get_num_active_clean:  expression+='not(contains(.,'\''stale'\''))'
       .../qa/workunits/ceph-helpers.sh:867: get_num_active_clean:  expression+=')'
       .../qa/workunits/ceph-helpers.sh:871: get_num_active_clean:  ceph --format xml pg dump pgs
       .../qa/workunits/ceph-helpers.sh:872: get_num_active_clean:  xmlstarlet sel -t -m '//pg_stat/state[(contains(.,'\''active'\'') and contains(.,'\''clean'\'') and not(contains(.,'\''stale'\'')))]' -v . -n
       .../qa/workunits/ceph-helpers.sh:873: get_num_active_clean:  grep -v '^$'
       .../qa/workunits/ceph-helpers.sh:873: get_num_active_clean:  wc -l
       ../qa/workunits/ceph-helpers.sh:954: is_clean:  test 0 = 4
       ../qa/workunits/ceph-helpers.sh:954: is_clean:  return 1
       ../qa/workunits/ceph-helpers.sh:986: wait_for_clean:  get_is_making_recovery_progress
       .../qa/workunits/ceph-helpers.sh:839: get_is_making_recovery_progress:  ceph --format xml status
       .../qa/workunits/ceph-helpers.sh:839: get_is_making_recovery_progress:  xmlstarlet sel -t -m //pgmap/recovering_keys_per_sec -v . -o ' ' -t -m //pgmap/recovering_bytes_per_sec -v . -o ' ' -t -m //pgmap/recovering_objects_per_sec -v .
       ../qa/workunits/ceph-helpers.sh:839: get_is_making_recovery_progress:  local progress=
       ../qa/workunits/ceph-helpers.sh:840: get_is_making_recovery_progress:  test -n ''
       ../qa/workunits/ceph-helpers.sh:988: wait_for_clean:  ((  timer >= 120  ))
       .../qa/workunits/ceph-helpers.sh:993: wait_for_clean:  get_num_active_clean
       .../qa/workunits/ceph-helpers.sh:863: get_num_active_clean:  local 'expression=('
       .../qa/workunits/ceph-helpers.sh:864: get_num_active_clean:  expression+='contains(.,'\''active'\'') and '
       .../qa/workunits/ceph-helpers.sh:865: get_num_active_clean:  expression+='contains(.,'\''clean'\'') and '
       .../qa/workunits/ceph-helpers.sh:866: get_num_active_clean:  expression+='not(contains(.,'\''stale'\''))'
       .../qa/workunits/ceph-helpers.sh:867: get_num_active_clean:  expression+=')'
       .../qa/workunits/ceph-helpers.sh:871: get_num_active_clean:  ceph --format xml pg dump pgs
       .../qa/workunits/ceph-helpers.sh:872: get_num_active_clean:  xmlstarlet sel -t -m '//pg_stat/state[(contains(.,'\''active'\'') and contains(.,'\''clean'\'') and not(contains(.,'\''stale'\'')))]' -v . -n
       .../qa/workunits/ceph-helpers.sh:873: get_num_active_clean:  grep -v '^$'
       .../qa/workunits/ceph-helpers.sh:873: get_num_active_clean:  wc -l
       ../qa/workunits/ceph-helpers.sh:993: wait_for_clean:  cur_active_clean=0
       ../qa/workunits/ceph-helpers.sh:994: wait_for_clean:  test 0 '!=' 0
       ../qa/workunits/ceph-helpers.sh:998: wait_for_clean:  sleep 1
       .../qa/workunits/ceph-helpers.sh:999: wait_for_clean:  expr 106 + 1
       ../qa/workunits/ceph-helpers.sh:999: wait_for_clean:  timer=107
       ../qa/workunits/ceph-helpers.sh:985: wait_for_clean:  is_clean
       .../qa/workunits/ceph-helpers.sh:952: is_clean:  get_num_pgs
       .../qa/workunits/ceph-helpers.sh:898: get_num_pgs:  ceph --format xml status
       .../qa/workunits/ceph-helpers.sh:899: get_num_pgs:  xmlstarlet sel -t -m //pgmap/num_pgs -v .
       ../qa/workunits/ceph-helpers.sh:952: is_clean:  num_pgs=4
       ../qa/workunits/ceph-helpers.sh:953: is_clean:  test 4 '!=' 0
       .../qa/workunits/ceph-helpers.sh:954: is_clean:  get_num_active_clean
       .../qa/workunits/ceph-helpers.sh:863: get_num_active_clean:  local 'expression=('
       .../qa/workunits/ceph-helpers.sh:864: get_num_active_clean:  expression+='contains(.,'\''active'\'') and '
       .../qa/workunits/ceph-helpers.sh:865: get_num_active_clean:  expression+='contains(.,'\''clean'\'') and '
       .../qa/workunits/ceph-helpers.sh:866: get_num_active_clean:  expression+='not(contains(.,'\''stale'\''))'
       .../qa/workunits/ceph-helpers.sh:867: get_num_active_clean:  expression+=')'
       .../qa/workunits/ceph-helpers.sh:871: get_num_active_clean:  ceph --format xml pg dump pgs
       .../qa/workunits/ceph-helpers.sh:872: get_num_active_clean:  xmlstarlet sel -t -m '//pg_stat/state[(contains(.,'\''active'\'') and contains(.,'\''clean'\'') and not(contains(.,'\''stale'\'')))]' -v . -n
       .../qa/workunits/ceph-helpers.sh:873: get_num_active_clean:  grep -v '^$'
       .../qa/workunits/ceph-helpers.sh:873: get_num_active_clean:  wc -l
       ../qa/workunits/ceph-helpers.sh:954: is_clean:  test 0 = 4
       ../qa/workunits/ceph-helpers.sh:954: is_clean:  return 1
       ../qa/workunits/ceph-helpers.sh:986: wait_for_clean:  get_is_making_recovery_progress
       .../qa/workunits/ceph-helpers.sh:839: get_is_making_recovery_progress:  ceph --format xml status
       .../qa/workunits/ceph-helpers.sh:839: get_is_making_recovery_progress:  xmlstarlet sel -t -m //pgmap/recovering_keys_per_sec -v . -o ' ' -t -m //pgmap/recovering_bytes_per_sec -v . -o ' ' -t -m //pgmap/recovering_objects_per_sec -v .
       ../qa/workunits/ceph-helpers.sh:839: get_is_making_recovery_progress:  local progress=
       ../qa/workunits/ceph-helpers.sh:840: get_is_making_recovery_progress:  test -n ''
       ../qa/workunits/ceph-helpers.sh:988: wait_for_clean:  ((  timer >= 120  ))
       .../qa/workunits/ceph-helpers.sh:993: wait_for_clean:  get_num_active_clean
       .../qa/workunits/ceph-helpers.sh:863: get_num_active_clean:  local 'expression=('
       .../qa/workunits/ceph-helpers.sh:864: get_num_active_clean:  expression+='contains(.,'\''active'\'') and '
       .../qa/workunits/ceph-helpers.sh:865: get_num_active_clean:  expression+='contains(.,'\''clean'\'') and '
       .../qa/workunits/ceph-helpers.sh:866: get_num_active_clean:  expression+='not(contains(.,'\''stale'\''))'
       .../qa/workunits/ceph-helpers.sh:867: get_num_active_clean:  expression+=')'
       .../qa/workunits/ceph-helpers.sh:871: get_num_active_clean:  ceph --format xml pg dump pgs
       .../qa/workunits/ceph-helpers.sh:872: get_num_active_clean:  xmlstarlet sel -t -m '//pg_stat/state[(contains(.,'\''active'\'') and contains(.,'\''clean'\'') and not(contains(.,'\''stale'\'')))]' -v . -n
       .../qa/workunits/ceph-helpers.sh:873: get_num_active_clean:  grep -v '^$'
       .../qa/workunits/ceph-helpers.sh:873: get_num_active_clean:  wc -l
       ../qa/workunits/ceph-helpers.sh:993: wait_for_clean:  cur_active_clean=0
       ../qa/workunits/ceph-helpers.sh:994: wait_for_clean:  test 0 '!=' 0
       ../qa/workunits/ceph-helpers.sh:998: wait_for_clean:  sleep 1
       .../qa/workunits/ceph-helpers.sh:999: wait_for_clean:  expr 107 + 1
       ../qa/workunits/ceph-helpers.sh:999: wait_for_clean:  timer=108
       ../qa/workunits/ceph-helpers.sh:985: wait_for_clean:  is_clean
       .../qa/workunits/ceph-helpers.sh:952: is_clean:  get_num_pgs
       .../qa/workunits/ceph-helpers.sh:898: get_num_pgs:  ceph --format xml status
       .../qa/workunits/ceph-helpers.sh:899: get_num_pgs:  xmlstarlet sel -t -m //pgmap/num_pgs -v .
       ../qa/workunits/ceph-helpers.sh:952: is_clean:  num_pgs=4
       ../qa/workunits/ceph-helpers.sh:953: is_clean:  test 4 '!=' 0
       .../qa/workunits/ceph-helpers.sh:954: is_clean:  get_num_active_clean
       .../qa/workunits/ceph-helpers.sh:863: get_num_active_clean:  local 'expression=('
       .../qa/workunits/ceph-helpers.sh:864: get_num_active_clean:  expression+='contains(.,'\''active'\'') and '
       .../qa/workunits/ceph-helpers.sh:865: get_num_active_clean:  expression+='contains(.,'\''clean'\'') and '
       .../qa/workunits/ceph-helpers.sh:866: get_num_active_clean:  expression+='not(contains(.,'\''stale'\''))'
       .../qa/workunits/ceph-helpers.sh:867: get_num_active_clean:  expression+=')'
       .../qa/workunits/ceph-helpers.sh:871: get_num_active_clean:  ceph --format xml pg dump pgs
       .../qa/workunits/ceph-helpers.sh:872: get_num_active_clean:  xmlstarlet sel -t -m '//pg_stat/state[(contains(.,'\''active'\'') and contains(.,'\''clean'\'') and not(contains(.,'\''stale'\'')))]' -v . -n
       .../qa/workunits/ceph-helpers.sh:873: get_num_active_clean:  grep -v '^$'
       .../qa/workunits/ceph-helpers.sh:873: get_num_active_clean:  wc -l
       ../qa/workunits/ceph-helpers.sh:954: is_clean:  test 0 = 4
       ../qa/workunits/ceph-helpers.sh:954: is_clean:  return 1
       ../qa/workunits/ceph-helpers.sh:986: wait_for_clean:  get_is_making_recovery_progress
       .../qa/workunits/ceph-helpers.sh:839: get_is_making_recovery_progress:  ceph --format xml status
       .../qa/workunits/ceph-helpers.sh:839: get_is_making_recovery_progress:  xmlstarlet sel -t -m //pgmap/recovering_keys_per_sec -v . -o ' ' -t -m //pgmap/recovering_bytes_per_sec -v . -o ' ' -t -m //pgmap/recovering_objects_per_sec -v .
       ../qa/workunits/ceph-helpers.sh:839: get_is_making_recovery_progress:  local progress=
       ../qa/workunits/ceph-helpers.sh:840: get_is_making_recovery_progress:  test -n ''
       ../qa/workunits/ceph-helpers.sh:988: wait_for_clean:  ((  timer >= 120  ))
       .../qa/workunits/ceph-helpers.sh:993: wait_for_clean:  get_num_active_clean
       .../qa/workunits/ceph-helpers.sh:863: get_num_active_clean:  local 'expression=('
       .../qa/workunits/ceph-helpers.sh:864: get_num_active_clean:  expression+='contains(.,'\''active'\'') and '
       .../qa/workunits/ceph-helpers.sh:865: get_num_active_clean:  expression+='contains(.,'\''clean'\'') and '
       .../qa/workunits/ceph-helpers.sh:866: get_num_active_clean:  expression+='not(contains(.,'\''stale'\''))'
       .../qa/workunits/ceph-helpers.sh:867: get_num_active_clean:  expression+=')'
       .../qa/workunits/ceph-helpers.sh:871: get_num_active_clean:  ceph --format xml pg dump pgs
       .../qa/workunits/ceph-helpers.sh:872: get_num_active_clean:  xmlstarlet sel -t -m '//pg_stat/state[(contains(.,'\''active'\'') and contains(.,'\''clean'\'') and not(contains(.,'\''stale'\'')))]' -v . -n
       .../qa/workunits/ceph-helpers.sh:873: get_num_active_clean:  grep -v '^$'
       .../qa/workunits/ceph-helpers.sh:873: get_num_active_clean:  wc -l
       ../qa/workunits/ceph-helpers.sh:993: wait_for_clean:  cur_active_clean=0
       ../qa/workunits/ceph-helpers.sh:994: wait_for_clean:  test 0 '!=' 0
       ../qa/workunits/ceph-helpers.sh:998: wait_for_clean:  sleep 1
       .../qa/workunits/ceph-helpers.sh:999: wait_for_clean:  expr 108 + 1
       ../qa/workunits/ceph-helpers.sh:999: wait_for_clean:  timer=109
       ../qa/workunits/ceph-helpers.sh:985: wait_for_clean:  is_clean
       .../qa/workunits/ceph-helpers.sh:952: is_clean:  get_num_pgs
       .../qa/workunits/ceph-helpers.sh:898: get_num_pgs:  ceph --format xml status
       .../qa/workunits/ceph-helpers.sh:899: get_num_pgs:  xmlstarlet sel -t -m //pgmap/num_pgs -v .
       ../qa/workunits/ceph-helpers.sh:952: is_clean:  num_pgs=4
       ../qa/workunits/ceph-helpers.sh:953: is_clean:  test 4 '!=' 0
       .../qa/workunits/ceph-helpers.sh:954: is_clean:  get_num_active_clean
       .../qa/workunits/ceph-helpers.sh:863: get_num_active_clean:  local 'expression=('
       .../qa/workunits/ceph-helpers.sh:864: get_num_active_clean:  expression+='contains(.,'\''active'\'') and '
       .../qa/workunits/ceph-helpers.sh:865: get_num_active_clean:  expression+='contains(.,'\''clean'\'') and '
       .../qa/workunits/ceph-helpers.sh:866: get_num_active_clean:  expression+='not(contains(.,'\''stale'\''))'
       .../qa/workunits/ceph-helpers.sh:867: get_num_active_clean:  expression+=')'
       .../qa/workunits/ceph-helpers.sh:871: get_num_active_clean:  ceph --format xml pg dump pgs
       .../qa/workunits/ceph-helpers.sh:872: get_num_active_clean:  xmlstarlet sel -t -m '//pg_stat/state[(contains(.,'\''active'\'') and contains(.,'\''clean'\'') and not(contains(.,'\''stale'\'')))]' -v . -n
       .../qa/workunits/ceph-helpers.sh:873: get_num_active_clean:  grep -v '^$'
       .../qa/workunits/ceph-helpers.sh:873: get_num_active_clean:  wc -l
       ../qa/workunits/ceph-helpers.sh:954: is_clean:  test 0 = 4
       ../qa/workunits/ceph-helpers.sh:954: is_clean:  return 1
       ../qa/workunits/ceph-helpers.sh:986: wait_for_clean:  get_is_making_recovery_progress
       .../qa/workunits/ceph-helpers.sh:839: get_is_making_recovery_progress:  ceph --format xml status
       .../qa/workunits/ceph-helpers.sh:839: get_is_making_recovery_progress:  xmlstarlet sel -t -m //pgmap/recovering_keys_per_sec -v . -o ' ' -t -m //pgmap/recovering_bytes_per_sec -v . -o ' ' -t -m //pgmap/recovering_objects_per_sec -v .
       ../qa/workunits/ceph-helpers.sh:839: get_is_making_recovery_progress:  local progress=
       ../qa/workunits/ceph-helpers.sh:840: get_is_making_recovery_progress:  test -n ''
       ../qa/workunits/ceph-helpers.sh:988: wait_for_clean:  ((  timer >= 120  ))
       .../qa/workunits/ceph-helpers.sh:993: wait_for_clean:  get_num_active_clean
       .../qa/workunits/ceph-helpers.sh:863: get_num_active_clean:  local 'expression=('
       .../qa/workunits/ceph-helpers.sh:864: get_num_active_clean:  expression+='contains(.,'\''active'\'') and '
       .../qa/workunits/ceph-helpers.sh:865: get_num_active_clean:  expression+='contains(.,'\''clean'\'') and '
       .../qa/workunits/ceph-helpers.sh:866: get_num_active_clean:  expression+='not(contains(.,'\''stale'\''))'
       .../qa/workunits/ceph-helpers.sh:867: get_num_active_clean:  expression+=')'
       .../qa/workunits/ceph-helpers.sh:871: get_num_active_clean:  ceph --format xml pg dump pgs
       .../qa/workunits/ceph-helpers.sh:872: get_num_active_clean:  xmlstarlet sel -t -m '//pg_stat/state[(contains(.,'\''active'\'') and contains(.,'\''clean'\'') and not(contains(.,'\''stale'\'')))]' -v . -n
       .../qa/workunits/ceph-helpers.sh:873: get_num_active_clean:  wc -l
       .../qa/workunits/ceph-helpers.sh:873: get_num_active_clean:  grep -v '^$'
       ../qa/workunits/ceph-helpers.sh:993: wait_for_clean:  cur_active_clean=0
       ../qa/workunits/ceph-helpers.sh:994: wait_for_clean:  test 0 '!=' 0
       ../qa/workunits/ceph-helpers.sh:998: wait_for_clean:  sleep 1
       .../qa/workunits/ceph-helpers.sh:999: wait_for_clean:  expr 109 + 1
       ../qa/workunits/ceph-helpers.sh:999: wait_for_clean:  timer=110
       ../qa/workunits/ceph-helpers.sh:985: wait_for_clean:  is_clean
       .../qa/workunits/ceph-helpers.sh:952: is_clean:  get_num_pgs
       .../qa/workunits/ceph-helpers.sh:898: get_num_pgs:  ceph --format xml status
       .../qa/workunits/ceph-helpers.sh:899: get_num_pgs:  xmlstarlet sel -t -m //pgmap/num_pgs -v .
       ../qa/workunits/ceph-helpers.sh:952: is_clean:  num_pgs=4
       ../qa/workunits/ceph-helpers.sh:953: is_clean:  test 4 '!=' 0
       .../qa/workunits/ceph-helpers.sh:954: is_clean:  get_num_active_clean
       .../qa/workunits/ceph-helpers.sh:863: get_num_active_clean:  local 'expression=('
       .../qa/workunits/ceph-helpers.sh:864: get_num_active_clean:  expression+='contains(.,'\''active'\'') and '
       .../qa/workunits/ceph-helpers.sh:865: get_num_active_clean:  expression+='contains(.,'\''clean'\'') and '
       .../qa/workunits/ceph-helpers.sh:866: get_num_active_clean:  expression+='not(contains(.,'\''stale'\''))'
       .../qa/workunits/ceph-helpers.sh:867: get_num_active_clean:  expression+=')'
       .../qa/workunits/ceph-helpers.sh:871: get_num_active_clean:  ceph --format xml pg dump pgs
       .../qa/workunits/ceph-helpers.sh:872: get_num_active_clean:  xmlstarlet sel -t -m '//pg_stat/state[(contains(.,'\''active'\'') and contains(.,'\''clean'\'') and not(contains(.,'\''stale'\'')))]' -v . -n
       .../qa/workunits/ceph-helpers.sh:873: get_num_active_clean:  grep -v '^$'
       .../qa/workunits/ceph-helpers.sh:873: get_num_active_clean:  wc -l
       ../qa/workunits/ceph-helpers.sh:954: is_clean:  test 0 = 4
       ../qa/workunits/ceph-helpers.sh:954: is_clean:  return 1
       ../qa/workunits/ceph-helpers.sh:986: wait_for_clean:  get_is_making_recovery_progress
       .../qa/workunits/ceph-helpers.sh:839: get_is_making_recovery_progress:  ceph --format xml status
       .../qa/workunits/ceph-helpers.sh:839: get_is_making_recovery_progress:  xmlstarlet sel -t -m //pgmap/recovering_keys_per_sec -v . -o ' ' -t -m //pgmap/recovering_bytes_per_sec -v . -o ' ' -t -m //pgmap/recovering_objects_per_sec -v .
       ../qa/workunits/ceph-helpers.sh:839: get_is_making_recovery_progress:  local progress=
       ../qa/workunits/ceph-helpers.sh:840: get_is_making_recovery_progress:  test -n ''
       ../qa/workunits/ceph-helpers.sh:988: wait_for_clean:  ((  timer >= 120  ))
       .../qa/workunits/ceph-helpers.sh:993: wait_for_clean:  get_num_active_clean
       .../qa/workunits/ceph-helpers.sh:863: get_num_active_clean:  local 'expression=('
       .../qa/workunits/ceph-helpers.sh:864: get_num_active_clean:  expression+='contains(.,'\''active'\'') and '
       .../qa/workunits/ceph-helpers.sh:865: get_num_active_clean:  expression+='contains(.,'\''clean'\'') and '
       .../qa/workunits/ceph-helpers.sh:866: get_num_active_clean:  expression+='not(contains(.,'\''stale'\''))'
       .../qa/workunits/ceph-helpers.sh:867: get_num_active_clean:  expression+=')'
       .../qa/workunits/ceph-helpers.sh:871: get_num_active_clean:  ceph --format xml pg dump pgs
       .../qa/workunits/ceph-helpers.sh:872: get_num_active_clean:  xmlstarlet sel -t -m '//pg_stat/state[(contains(.,'\''active'\'') and contains(.,'\''clean'\'') and not(contains(.,'\''stale'\'')))]' -v . -n
       .../qa/workunits/ceph-helpers.sh:873: get_num_active_clean:  grep -v '^$'
       .../qa/workunits/ceph-helpers.sh:873: get_num_active_clean:  wc -l
       ../qa/workunits/ceph-helpers.sh:993: wait_for_clean:  cur_active_clean=0
       ../qa/workunits/ceph-helpers.sh:994: wait_for_clean:  test 0 '!=' 0
       ../qa/workunits/ceph-helpers.sh:998: wait_for_clean:  sleep 1
       .../qa/workunits/ceph-helpers.sh:999: wait_for_clean:  expr 110 + 1
       ../qa/workunits/ceph-helpers.sh:999: wait_for_clean:  timer=111
       ../qa/workunits/ceph-helpers.sh:985: wait_for_clean:  is_clean
       .../qa/workunits/ceph-helpers.sh:952: is_clean:  get_num_pgs
       .../qa/workunits/ceph-helpers.sh:898: get_num_pgs:  ceph --format xml status
       .../qa/workunits/ceph-helpers.sh:899: get_num_pgs:  xmlstarlet sel -t -m //pgmap/num_pgs -v .
       ../qa/workunits/ceph-helpers.sh:952: is_clean:  num_pgs=4
       ../qa/workunits/ceph-helpers.sh:953: is_clean:  test 4 '!=' 0
       .../qa/workunits/ceph-helpers.sh:954: is_clean:  get_num_active_clean
       .../qa/workunits/ceph-helpers.sh:863: get_num_active_clean:  local 'expression=('
       .../qa/workunits/ceph-helpers.sh:864: get_num_active_clean:  expression+='contains(.,'\''active'\'') and '
       .../qa/workunits/ceph-helpers.sh:865: get_num_active_clean:  expression+='contains(.,'\''clean'\'') and '
       .../qa/workunits/ceph-helpers.sh:866: get_num_active_clean:  expression+='not(contains(.,'\''stale'\''))'
       .../qa/workunits/ceph-helpers.sh:867: get_num_active_clean:  expression+=')'
       .../qa/workunits/ceph-helpers.sh:871: get_num_active_clean:  ceph --format xml pg dump pgs
       .../qa/workunits/ceph-helpers.sh:872: get_num_active_clean:  xmlstarlet sel -t -m '//pg_stat/state[(contains(.,'\''active'\'') and contains(.,'\''clean'\'') and not(contains(.,'\''stale'\'')))]' -v . -n
       .../qa/workunits/ceph-helpers.sh:873: get_num_active_clean:  grep -v '^$'
       .../qa/workunits/ceph-helpers.sh:873: get_num_active_clean:  wc -l
       ../qa/workunits/ceph-helpers.sh:954: is_clean:  test 0 = 4
       ../qa/workunits/ceph-helpers.sh:954: is_clean:  return 1
       ../qa/workunits/ceph-helpers.sh:986: wait_for_clean:  get_is_making_recovery_progress
       .../qa/workunits/ceph-helpers.sh:839: get_is_making_recovery_progress:  ceph --format xml status
       .../qa/workunits/ceph-helpers.sh:839: get_is_making_recovery_progress:  xmlstarlet sel -t -m //pgmap/recovering_keys_per_sec -v . -o ' ' -t -m //pgmap/recovering_bytes_per_sec -v . -o ' ' -t -m //pgmap/recovering_objects_per_sec -v .
       ../qa/workunits/ceph-helpers.sh:839: get_is_making_recovery_progress:  local progress=
       ../qa/workunits/ceph-helpers.sh:840: get_is_making_recovery_progress:  test -n ''
       ../qa/workunits/ceph-helpers.sh:988: wait_for_clean:  ((  timer >= 120  ))
       .../qa/workunits/ceph-helpers.sh:993: wait_for_clean:  get_num_active_clean
       .../qa/workunits/ceph-helpers.sh:863: get_num_active_clean:  local 'expression=('
       .../qa/workunits/ceph-helpers.sh:864: get_num_active_clean:  expression+='contains(.,'\''active'\'') and '
       .../qa/workunits/ceph-helpers.sh:865: get_num_active_clean:  expression+='contains(.,'\''clean'\'') and '
       .../qa/workunits/ceph-helpers.sh:866: get_num_active_clean:  expression+='not(contains(.,'\''stale'\''))'
       .../qa/workunits/ceph-helpers.sh:867: get_num_active_clean:  expression+=')'
       .../qa/workunits/ceph-helpers.sh:871: get_num_active_clean:  ceph --format xml pg dump pgs
       .../qa/workunits/ceph-helpers.sh:872: get_num_active_clean:  xmlstarlet sel -t -m '//pg_stat/state[(contains(.,'\''active'\'') and contains(.,'\''clean'\'') and not(contains(.,'\''stale'\'')))]' -v . -n
       .../qa/workunits/ceph-helpers.sh:873: get_num_active_clean:  grep -v '^$'
       .../qa/workunits/ceph-helpers.sh:873: get_num_active_clean:  wc -l
       ../qa/workunits/ceph-helpers.sh:993: wait_for_clean:  cur_active_clean=0
       ../qa/workunits/ceph-helpers.sh:994: wait_for_clean:  test 0 '!=' 0
       ../qa/workunits/ceph-helpers.sh:998: wait_for_clean:  sleep 1
       .../qa/workunits/ceph-helpers.sh:999: wait_for_clean:  expr 111 + 1
       ../qa/workunits/ceph-helpers.sh:999: wait_for_clean:  timer=112
       ../qa/workunits/ceph-helpers.sh:985: wait_for_clean:  is_clean
       .../qa/workunits/ceph-helpers.sh:952: is_clean:  get_num_pgs
       .../qa/workunits/ceph-helpers.sh:898: get_num_pgs:  ceph --format xml status
       .../qa/workunits/ceph-helpers.sh:899: get_num_pgs:  xmlstarlet sel -t -m //pgmap/num_pgs -v .
       ../qa/workunits/ceph-helpers.sh:952: is_clean:  num_pgs=4
       ../qa/workunits/ceph-helpers.sh:953: is_clean:  test 4 '!=' 0
       .../qa/workunits/ceph-helpers.sh:954: is_clean:  get_num_active_clean
       .../qa/workunits/ceph-helpers.sh:863: get_num_active_clean:  local 'expression=('
       .../qa/workunits/ceph-helpers.sh:864: get_num_active_clean:  expression+='contains(.,'\''active'\'') and '
       .../qa/workunits/ceph-helpers.sh:865: get_num_active_clean:  expression+='contains(.,'\''clean'\'') and '
       .../qa/workunits/ceph-helpers.sh:866: get_num_active_clean:  expression+='not(contains(.,'\''stale'\''))'
       .../qa/workunits/ceph-helpers.sh:867: get_num_active_clean:  expression+=')'
       .../qa/workunits/ceph-helpers.sh:871: get_num_active_clean:  ceph --format xml pg dump pgs
       .../qa/workunits/ceph-helpers.sh:872: get_num_active_clean:  xmlstarlet sel -t -m '//pg_stat/state[(contains(.,'\''active'\'') and contains(.,'\''clean'\'') and not(contains(.,'\''stale'\'')))]' -v . -n
       .../qa/workunits/ceph-helpers.sh:873: get_num_active_clean:  wc -l
       .../qa/workunits/ceph-helpers.sh:873: get_num_active_clean:  grep -v '^$'
       ../qa/workunits/ceph-helpers.sh:954: is_clean:  test 0 = 4
       ../qa/workunits/ceph-helpers.sh:954: is_clean:  return 1
       ../qa/workunits/ceph-helpers.sh:986: wait_for_clean:  get_is_making_recovery_progress
       .../qa/workunits/ceph-helpers.sh:839: get_is_making_recovery_progress:  ceph --format xml status
       .../qa/workunits/ceph-helpers.sh:839: get_is_making_recovery_progress:  xmlstarlet sel -t -m //pgmap/recovering_keys_per_sec -v . -o ' ' -t -m //pgmap/recovering_bytes_per_sec -v . -o ' ' -t -m //pgmap/recovering_objects_per_sec -v .
       ../qa/workunits/ceph-helpers.sh:839: get_is_making_recovery_progress:  local progress=
       ../qa/workunits/ceph-helpers.sh:840: get_is_making_recovery_progress:  test -n ''
       ../qa/workunits/ceph-helpers.sh:988: wait_for_clean:  ((  timer >= 120  ))
       .../qa/workunits/ceph-helpers.sh:993: wait_for_clean:  get_num_active_clean
       .../qa/workunits/ceph-helpers.sh:863: get_num_active_clean:  local 'expression=('
       .../qa/workunits/ceph-helpers.sh:864: get_num_active_clean:  expression+='contains(.,'\''active'\'') and '
       .../qa/workunits/ceph-helpers.sh:865: get_num_active_clean:  expression+='contains(.,'\''clean'\'') and '
       .../qa/workunits/ceph-helpers.sh:866: get_num_active_clean:  expression+='not(contains(.,'\''stale'\''))'
       .../qa/workunits/ceph-helpers.sh:867: get_num_active_clean:  expression+=')'
       .../qa/workunits/ceph-helpers.sh:871: get_num_active_clean:  ceph --format xml pg dump pgs
       .../qa/workunits/ceph-helpers.sh:872: get_num_active_clean:  xmlstarlet sel -t -m '//pg_stat/state[(contains(.,'\''active'\'') and contains(.,'\''clean'\'') and not(contains(.,'\''stale'\'')))]' -v . -n
       .../qa/workunits/ceph-helpers.sh:873: get_num_active_clean:  wc -l
       .../qa/workunits/ceph-helpers.sh:873: get_num_active_clean:  grep -v '^$'
       ../qa/workunits/ceph-helpers.sh:993: wait_for_clean:  cur_active_clean=0
       ../qa/workunits/ceph-helpers.sh:994: wait_for_clean:  test 0 '!=' 0
       ../qa/workunits/ceph-helpers.sh:998: wait_for_clean:  sleep 1
       .../qa/workunits/ceph-helpers.sh:999: wait_for_clean:  expr 112 + 1
       ../qa/workunits/ceph-helpers.sh:999: wait_for_clean:  timer=113
       ../qa/workunits/ceph-helpers.sh:985: wait_for_clean:  is_clean
       .../qa/workunits/ceph-helpers.sh:952: is_clean:  get_num_pgs
       .../qa/workunits/ceph-helpers.sh:898: get_num_pgs:  ceph --format xml status
       .../qa/workunits/ceph-helpers.sh:899: get_num_pgs:  xmlstarlet sel -t -m //pgmap/num_pgs -v .
       ../qa/workunits/ceph-helpers.sh:952: is_clean:  num_pgs=4
       ../qa/workunits/ceph-helpers.sh:953: is_clean:  test 4 '!=' 0
       .../qa/workunits/ceph-helpers.sh:954: is_clean:  get_num_active_clean
       .../qa/workunits/ceph-helpers.sh:863: get_num_active_clean:  local 'expression=('
       .../qa/workunits/ceph-helpers.sh:864: get_num_active_clean:  expression+='contains(.,'\''active'\'') and '
       .../qa/workunits/ceph-helpers.sh:865: get_num_active_clean:  expression+='contains(.,'\''clean'\'') and '
       .../qa/workunits/ceph-helpers.sh:866: get_num_active_clean:  expression+='not(contains(.,'\''stale'\''))'
       .../qa/workunits/ceph-helpers.sh:867: get_num_active_clean:  expression+=')'
       .../qa/workunits/ceph-helpers.sh:871: get_num_active_clean:  ceph --format xml pg dump pgs
       .../qa/workunits/ceph-helpers.sh:872: get_num_active_clean:  xmlstarlet sel -t -m '//pg_stat/state[(contains(.,'\''active'\'') and contains(.,'\''clean'\'') and not(contains(.,'\''stale'\'')))]' -v . -n
       .../qa/workunits/ceph-helpers.sh:873: get_num_active_clean:  grep -v '^$'
       .../qa/workunits/ceph-helpers.sh:873: get_num_active_clean:  wc -l
       ../qa/workunits/ceph-helpers.sh:954: is_clean:  test 0 = 4
       ../qa/workunits/ceph-helpers.sh:954: is_clean:  return 1
       ../qa/workunits/ceph-helpers.sh:986: wait_for_clean:  get_is_making_recovery_progress
       .../qa/workunits/ceph-helpers.sh:839: get_is_making_recovery_progress:  ceph --format xml status
       .../qa/workunits/ceph-helpers.sh:839: get_is_making_recovery_progress:  xmlstarlet sel -t -m //pgmap/recovering_keys_per_sec -v . -o ' ' -t -m //pgmap/recovering_bytes_per_sec -v . -o ' ' -t -m //pgmap/recovering_objects_per_sec -v .
       ../qa/workunits/ceph-helpers.sh:839: get_is_making_recovery_progress:  local progress=
       ../qa/workunits/ceph-helpers.sh:840: get_is_making_recovery_progress:  test -n ''
       ../qa/workunits/ceph-helpers.sh:988: wait_for_clean:  ((  timer >= 120  ))
       .../qa/workunits/ceph-helpers.sh:993: wait_for_clean:  get_num_active_clean
       .../qa/workunits/ceph-helpers.sh:863: get_num_active_clean:  local 'expression=('
       .../qa/workunits/ceph-helpers.sh:864: get_num_active_clean:  expression+='contains(.,'\''active'\'') and '
       .../qa/workunits/ceph-helpers.sh:865: get_num_active_clean:  expression+='contains(.,'\''clean'\'') and '
       .../qa/workunits/ceph-helpers.sh:866: get_num_active_clean:  expression+='not(contains(.,'\''stale'\''))'
       .../qa/workunits/ceph-helpers.sh:867: get_num_active_clean:  expression+=')'
       .../qa/workunits/ceph-helpers.sh:871: get_num_active_clean:  ceph --format xml pg dump pgs
       .../qa/workunits/ceph-helpers.sh:872: get_num_active_clean:  xmlstarlet sel -t -m '//pg_stat/state[(contains(.,'\''active'\'') and contains(.,'\''clean'\'') and not(contains(.,'\''stale'\'')))]' -v . -n
       .../qa/workunits/ceph-helpers.sh:873: get_num_active_clean:  grep -v '^$'
       .../qa/workunits/ceph-helpers.sh:873: get_num_active_clean:  wc -l
       ../qa/workunits/ceph-helpers.sh:993: wait_for_clean:  cur_active_clean=0
       ../qa/workunits/ceph-helpers.sh:994: wait_for_clean:  test 0 '!=' 0
       ../qa/workunits/ceph-helpers.sh:998: wait_for_clean:  sleep 1
       .../qa/workunits/ceph-helpers.sh:999: wait_for_clean:  expr 113 + 1
       ../qa/workunits/ceph-helpers.sh:999: wait_for_clean:  timer=114
       ../qa/workunits/ceph-helpers.sh:985: wait_for_clean:  is_clean
       .../qa/workunits/ceph-helpers.sh:952: is_clean:  get_num_pgs
       .../qa/workunits/ceph-helpers.sh:898: get_num_pgs:  ceph --format xml status
       .../qa/workunits/ceph-helpers.sh:899: get_num_pgs:  xmlstarlet sel -t -m //pgmap/num_pgs -v .
       ../qa/workunits/ceph-helpers.sh:952: is_clean:  num_pgs=4
       ../qa/workunits/ceph-helpers.sh:953: is_clean:  test 4 '!=' 0
       .../qa/workunits/ceph-helpers.sh:954: is_clean:  get_num_active_clean
       .../qa/workunits/ceph-helpers.sh:863: get_num_active_clean:  local 'expression=('
       .../qa/workunits/ceph-helpers.sh:864: get_num_active_clean:  expression+='contains(.,'\''active'\'') and '
       .../qa/workunits/ceph-helpers.sh:865: get_num_active_clean:  expression+='contains(.,'\''clean'\'') and '
       .../qa/workunits/ceph-helpers.sh:866: get_num_active_clean:  expression+='not(contains(.,'\''stale'\''))'
       .../qa/workunits/ceph-helpers.sh:867: get_num_active_clean:  expression+=')'
       .../qa/workunits/ceph-helpers.sh:871: get_num_active_clean:  ceph --format xml pg dump pgs
       .../qa/workunits/ceph-helpers.sh:872: get_num_active_clean:  xmlstarlet sel -t -m '//pg_stat/state[(contains(.,'\''active'\'') and contains(.,'\''clean'\'') and not(contains(.,'\''stale'\'')))]' -v . -n
       .../qa/workunits/ceph-helpers.sh:873: get_num_active_clean:  grep -v '^$'
       .../qa/workunits/ceph-helpers.sh:873: get_num_active_clean:  wc -l
       ../qa/workunits/ceph-helpers.sh:954: is_clean:  test 0 = 4
       ../qa/workunits/ceph-helpers.sh:954: is_clean:  return 1
       ../qa/workunits/ceph-helpers.sh:986: wait_for_clean:  get_is_making_recovery_progress
       .../qa/workunits/ceph-helpers.sh:839: get_is_making_recovery_progress:  ceph --format xml status
       .../qa/workunits/ceph-helpers.sh:839: get_is_making_recovery_progress:  xmlstarlet sel -t -m //pgmap/recovering_keys_per_sec -v . -o ' ' -t -m //pgmap/recovering_bytes_per_sec -v . -o ' ' -t -m //pgmap/recovering_objects_per_sec -v .
       ../qa/workunits/ceph-helpers.sh:839: get_is_making_recovery_progress:  local progress=
       ../qa/workunits/ceph-helpers.sh:840: get_is_making_recovery_progress:  test -n ''
       ../qa/workunits/ceph-helpers.sh:988: wait_for_clean:  ((  timer >= 120  ))
       .../qa/workunits/ceph-helpers.sh:993: wait_for_clean:  get_num_active_clean
       .../qa/workunits/ceph-helpers.sh:863: get_num_active_clean:  local 'expression=('
       .../qa/workunits/ceph-helpers.sh:864: get_num_active_clean:  expression+='contains(.,'\''active'\'') and '
       .../qa/workunits/ceph-helpers.sh:865: get_num_active_clean:  expression+='contains(.,'\''clean'\'') and '
       .../qa/workunits/ceph-helpers.sh:866: get_num_active_clean:  expression+='not(contains(.,'\''stale'\''))'
       .../qa/workunits/ceph-helpers.sh:867: get_num_active_clean:  expression+=')'
       .../qa/workunits/ceph-helpers.sh:871: get_num_active_clean:  ceph --format xml pg dump pgs
       .../qa/workunits/ceph-helpers.sh:872: get_num_active_clean:  xmlstarlet sel -t -m '//pg_stat/state[(contains(.,'\''active'\'') and contains(.,'\''clean'\'') and not(contains(.,'\''stale'\'')))]' -v . -n
       .../qa/workunits/ceph-helpers.sh:873: get_num_active_clean:  grep -v '^$'
       .../qa/workunits/ceph-helpers.sh:873: get_num_active_clean:  wc -l
       ../qa/workunits/ceph-helpers.sh:993: wait_for_clean:  cur_active_clean=0
       ../qa/workunits/ceph-helpers.sh:994: wait_for_clean:  test 0 '!=' 0
       ../qa/workunits/ceph-helpers.sh:998: wait_for_clean:  sleep 1
       .../qa/workunits/ceph-helpers.sh:999: wait_for_clean:  expr 114 + 1
       ../qa/workunits/ceph-helpers.sh:999: wait_for_clean:  timer=115
       ../qa/workunits/ceph-helpers.sh:985: wait_for_clean:  is_clean
       .../qa/workunits/ceph-helpers.sh:952: is_clean:  get_num_pgs
       .../qa/workunits/ceph-helpers.sh:898: get_num_pgs:  ceph --format xml status
       .../qa/workunits/ceph-helpers.sh:899: get_num_pgs:  xmlstarlet sel -t -m //pgmap/num_pgs -v .
       ../qa/workunits/ceph-helpers.sh:952: is_clean:  num_pgs=4
       ../qa/workunits/ceph-helpers.sh:953: is_clean:  test 4 '!=' 0
       .../qa/workunits/ceph-helpers.sh:954: is_clean:  get_num_active_clean
       .../qa/workunits/ceph-helpers.sh:863: get_num_active_clean:  local 'expression=('
       .../qa/workunits/ceph-helpers.sh:864: get_num_active_clean:  expression+='contains(.,'\''active'\'') and '
       .../qa/workunits/ceph-helpers.sh:865: get_num_active_clean:  expression+='contains(.,'\''clean'\'') and '
       .../qa/workunits/ceph-helpers.sh:866: get_num_active_clean:  expression+='not(contains(.,'\''stale'\''))'
       .../qa/workunits/ceph-helpers.sh:867: get_num_active_clean:  expression+=')'
       .../qa/workunits/ceph-helpers.sh:871: get_num_active_clean:  ceph --format xml pg dump pgs
       .../qa/workunits/ceph-helpers.sh:872: get_num_active_clean:  xmlstarlet sel -t -m '//pg_stat/state[(contains(.,'\''active'\'') and contains(.,'\''clean'\'') and not(contains(.,'\''stale'\'')))]' -v . -n
       .../qa/workunits/ceph-helpers.sh:873: get_num_active_clean:  wc -l
       .../qa/workunits/ceph-helpers.sh:873: get_num_active_clean:  grep -v '^$'
       ../qa/workunits/ceph-helpers.sh:954: is_clean:  test 0 = 4
       ../qa/workunits/ceph-helpers.sh:954: is_clean:  return 1
       ../qa/workunits/ceph-helpers.sh:986: wait_for_clean:  get_is_making_recovery_progress
       .../qa/workunits/ceph-helpers.sh:839: get_is_making_recovery_progress:  ceph --format xml status
       .../qa/workunits/ceph-helpers.sh:839: get_is_making_recovery_progress:  xmlstarlet sel -t -m //pgmap/recovering_keys_per_sec -v . -o ' ' -t -m //pgmap/recovering_bytes_per_sec -v . -o ' ' -t -m //pgmap/recovering_objects_per_sec -v .
       ../qa/workunits/ceph-helpers.sh:839: get_is_making_recovery_progress:  local progress=
       ../qa/workunits/ceph-helpers.sh:840: get_is_making_recovery_progress:  test -n ''
       ../qa/workunits/ceph-helpers.sh:988: wait_for_clean:  ((  timer >= 120  ))
       .../qa/workunits/ceph-helpers.sh:993: wait_for_clean:  get_num_active_clean
       .../qa/workunits/ceph-helpers.sh:863: get_num_active_clean:  local 'expression=('
       .../qa/workunits/ceph-helpers.sh:864: get_num_active_clean:  expression+='contains(.,'\''active'\'') and '
       .../qa/workunits/ceph-helpers.sh:865: get_num_active_clean:  expression+='contains(.,'\''clean'\'') and '
       .../qa/workunits/ceph-helpers.sh:866: get_num_active_clean:  expression+='not(contains(.,'\''stale'\''))'
       .../qa/workunits/ceph-helpers.sh:867: get_num_active_clean:  expression+=')'
       .../qa/workunits/ceph-helpers.sh:871: get_num_active_clean:  ceph --format xml pg dump pgs
       .../qa/workunits/ceph-helpers.sh:872: get_num_active_clean:  xmlstarlet sel -t -m '//pg_stat/state[(contains(.,'\''active'\'') and contains(.,'\''clean'\'') and not(contains(.,'\''stale'\'')))]' -v . -n
       .../qa/workunits/ceph-helpers.sh:873: get_num_active_clean:  grep -v '^$'
       .../qa/workunits/ceph-helpers.sh:873: get_num_active_clean:  wc -l
       ../qa/workunits/ceph-helpers.sh:993: wait_for_clean:  cur_active_clean=0
       ../qa/workunits/ceph-helpers.sh:994: wait_for_clean:  test 0 '!=' 0
       ../qa/workunits/ceph-helpers.sh:998: wait_for_clean:  sleep 1
       .../qa/workunits/ceph-helpers.sh:999: wait_for_clean:  expr 115 + 1
       ../qa/workunits/ceph-helpers.sh:999: wait_for_clean:  timer=116
       ../qa/workunits/ceph-helpers.sh:985: wait_for_clean:  is_clean
       .../qa/workunits/ceph-helpers.sh:952: is_clean:  get_num_pgs
       .../qa/workunits/ceph-helpers.sh:898: get_num_pgs:  ceph --format xml status
       .../qa/workunits/ceph-helpers.sh:899: get_num_pgs:  xmlstarlet sel -t -m //pgmap/num_pgs -v .
       ../qa/workunits/ceph-helpers.sh:952: is_clean:  num_pgs=4
       ../qa/workunits/ceph-helpers.sh:953: is_clean:  test 4 '!=' 0
       .../qa/workunits/ceph-helpers.sh:954: is_clean:  get_num_active_clean
       .../qa/workunits/ceph-helpers.sh:863: get_num_active_clean:  local 'expression=('
       .../qa/workunits/ceph-helpers.sh:864: get_num_active_clean:  expression+='contains(.,'\''active'\'') and '
       .../qa/workunits/ceph-helpers.sh:865: get_num_active_clean:  expression+='contains(.,'\''clean'\'') and '
       .../qa/workunits/ceph-helpers.sh:866: get_num_active_clean:  expression+='not(contains(.,'\''stale'\''))'
       .../qa/workunits/ceph-helpers.sh:867: get_num_active_clean:  expression+=')'
       .../qa/workunits/ceph-helpers.sh:871: get_num_active_clean:  ceph --format xml pg dump pgs
       .../qa/workunits/ceph-helpers.sh:872: get_num_active_clean:  xmlstarlet sel -t -m '//pg_stat/state[(contains(.,'\''active'\'') and contains(.,'\''clean'\'') and not(contains(.,'\''stale'\'')))]' -v . -n
       .../qa/workunits/ceph-helpers.sh:873: get_num_active_clean:  grep -v '^$'
       .../qa/workunits/ceph-helpers.sh:873: get_num_active_clean:  wc -l
       ../qa/workunits/ceph-helpers.sh:954: is_clean:  test 0 = 4
       ../qa/workunits/ceph-helpers.sh:954: is_clean:  return 1
       ../qa/workunits/ceph-helpers.sh:986: wait_for_clean:  get_is_making_recovery_progress
       .../qa/workunits/ceph-helpers.sh:839: get_is_making_recovery_progress:  ceph --format xml status
       .../qa/workunits/ceph-helpers.sh:839: get_is_making_recovery_progress:  xmlstarlet sel -t -m //pgmap/recovering_keys_per_sec -v . -o ' ' -t -m //pgmap/recovering_bytes_per_sec -v . -o ' ' -t -m //pgmap/recovering_objects_per_sec -v .
       ../qa/workunits/ceph-helpers.sh:839: get_is_making_recovery_progress:  local progress=
       ../qa/workunits/ceph-helpers.sh:840: get_is_making_recovery_progress:  test -n ''
       ../qa/workunits/ceph-helpers.sh:988: wait_for_clean:  ((  timer >= 120  ))
       .../qa/workunits/ceph-helpers.sh:993: wait_for_clean:  get_num_active_clean
       .../qa/workunits/ceph-helpers.sh:863: get_num_active_clean:  local 'expression=('
       .../qa/workunits/ceph-helpers.sh:864: get_num_active_clean:  expression+='contains(.,'\''active'\'') and '
       .../qa/workunits/ceph-helpers.sh:865: get_num_active_clean:  expression+='contains(.,'\''clean'\'') and '
       .../qa/workunits/ceph-helpers.sh:866: get_num_active_clean:  expression+='not(contains(.,'\''stale'\''))'
       .../qa/workunits/ceph-helpers.sh:867: get_num_active_clean:  expression+=')'
       .../qa/workunits/ceph-helpers.sh:871: get_num_active_clean:  ceph --format xml pg dump pgs
       .../qa/workunits/ceph-helpers.sh:872: get_num_active_clean:  xmlstarlet sel -t -m '//pg_stat/state[(contains(.,'\''active'\'') and contains(.,'\''clean'\'') and not(contains(.,'\''stale'\'')))]' -v . -n
       .../qa/workunits/ceph-helpers.sh:873: get_num_active_clean:  grep -v '^$'
       .../qa/workunits/ceph-helpers.sh:873: get_num_active_clean:  wc -l
       ../qa/workunits/ceph-helpers.sh:993: wait_for_clean:  cur_active_clean=0
       ../qa/workunits/ceph-helpers.sh:994: wait_for_clean:  test 0 '!=' 0
       ../qa/workunits/ceph-helpers.sh:998: wait_for_clean:  sleep 1
       .../qa/workunits/ceph-helpers.sh:999: wait_for_clean:  expr 116 + 1
       ../qa/workunits/ceph-helpers.sh:999: wait_for_clean:  timer=117
       ../qa/workunits/ceph-helpers.sh:985: wait_for_clean:  is_clean
       .../qa/workunits/ceph-helpers.sh:952: is_clean:  get_num_pgs
       .../qa/workunits/ceph-helpers.sh:898: get_num_pgs:  ceph --format xml status
       .../qa/workunits/ceph-helpers.sh:899: get_num_pgs:  xmlstarlet sel -t -m //pgmap/num_pgs -v .
       ../qa/workunits/ceph-helpers.sh:952: is_clean:  num_pgs=4
       ../qa/workunits/ceph-helpers.sh:953: is_clean:  test 4 '!=' 0
       .../qa/workunits/ceph-helpers.sh:954: is_clean:  get_num_active_clean
       .../qa/workunits/ceph-helpers.sh:863: get_num_active_clean:  local 'expression=('
       .../qa/workunits/ceph-helpers.sh:864: get_num_active_clean:  expression+='contains(.,'\''active'\'') and '
       .../qa/workunits/ceph-helpers.sh:865: get_num_active_clean:  expression+='contains(.,'\''clean'\'') and '
       .../qa/workunits/ceph-helpers.sh:866: get_num_active_clean:  expression+='not(contains(.,'\''stale'\''))'
       .../qa/workunits/ceph-helpers.sh:867: get_num_active_clean:  expression+=')'
       .../qa/workunits/ceph-helpers.sh:871: get_num_active_clean:  ceph --format xml pg dump pgs
       .../qa/workunits/ceph-helpers.sh:872: get_num_active_clean:  xmlstarlet sel -t -m '//pg_stat/state[(contains(.,'\''active'\'') and contains(.,'\''clean'\'') and not(contains(.,'\''stale'\'')))]' -v . -n
       .../qa/workunits/ceph-helpers.sh:873: get_num_active_clean:  grep -v '^$'
       .../qa/workunits/ceph-helpers.sh:873: get_num_active_clean:  wc -l
       ../qa/workunits/ceph-helpers.sh:954: is_clean:  test 0 = 4
       ../qa/workunits/ceph-helpers.sh:954: is_clean:  return 1
       ../qa/workunits/ceph-helpers.sh:986: wait_for_clean:  get_is_making_recovery_progress
       .../qa/workunits/ceph-helpers.sh:839: get_is_making_recovery_progress:  ceph --format xml status
       .../qa/workunits/ceph-helpers.sh:839: get_is_making_recovery_progress:  xmlstarlet sel -t -m //pgmap/recovering_keys_per_sec -v . -o ' ' -t -m //pgmap/recovering_bytes_per_sec -v . -o ' ' -t -m //pgmap/recovering_objects_per_sec -v .
       ../qa/workunits/ceph-helpers.sh:839: get_is_making_recovery_progress:  local progress=
       ../qa/workunits/ceph-helpers.sh:840: get_is_making_recovery_progress:  test -n ''
       ../qa/workunits/ceph-helpers.sh:988: wait_for_clean:  ((  timer >= 120  ))
       .../qa/workunits/ceph-helpers.sh:993: wait_for_clean:  get_num_active_clean
       .../qa/workunits/ceph-helpers.sh:863: get_num_active_clean:  local 'expression=('
       .../qa/workunits/ceph-helpers.sh:864: get_num_active_clean:  expression+='contains(.,'\''active'\'') and '
       .../qa/workunits/ceph-helpers.sh:865: get_num_active_clean:  expression+='contains(.,'\''clean'\'') and '
       .../qa/workunits/ceph-helpers.sh:866: get_num_active_clean:  expression+='not(contains(.,'\''stale'\''))'
       .../qa/workunits/ceph-helpers.sh:867: get_num_active_clean:  expression+=')'
       .../qa/workunits/ceph-helpers.sh:871: get_num_active_clean:  ceph --format xml pg dump pgs
       .../qa/workunits/ceph-helpers.sh:872: get_num_active_clean:  xmlstarlet sel -t -m '//pg_stat/state[(contains(.,'\''active'\'') and contains(.,'\''clean'\'') and not(contains(.,'\''stale'\'')))]' -v . -n
       .../qa/workunits/ceph-helpers.sh:873: get_num_active_clean:  grep -v '^$'
       .../qa/workunits/ceph-helpers.sh:873: get_num_active_clean:  wc -l
       ../qa/workunits/ceph-helpers.sh:993: wait_for_clean:  cur_active_clean=0
       ../qa/workunits/ceph-helpers.sh:994: wait_for_clean:  test 0 '!=' 0
       ../qa/workunits/ceph-helpers.sh:998: wait_for_clean:  sleep 1
       .../qa/workunits/ceph-helpers.sh:999: wait_for_clean:  expr 117 + 1
       ../qa/workunits/ceph-helpers.sh:999: wait_for_clean:  timer=118
       ../qa/workunits/ceph-helpers.sh:985: wait_for_clean:  is_clean
       .../qa/workunits/ceph-helpers.sh:952: is_clean:  get_num_pgs
       .../qa/workunits/ceph-helpers.sh:898: get_num_pgs:  ceph --format xml status
       .../qa/workunits/ceph-helpers.sh:899: get_num_pgs:  xmlstarlet sel -t -m //pgmap/num_pgs -v .
       ../qa/workunits/ceph-helpers.sh:952: is_clean:  num_pgs=4
       ../qa/workunits/ceph-helpers.sh:953: is_clean:  test 4 '!=' 0
       .../qa/workunits/ceph-helpers.sh:954: is_clean:  get_num_active_clean
       .../qa/workunits/ceph-helpers.sh:863: get_num_active_clean:  local 'expression=('
       .../qa/workunits/ceph-helpers.sh:864: get_num_active_clean:  expression+='contains(.,'\''active'\'') and '
       .../qa/workunits/ceph-helpers.sh:865: get_num_active_clean:  expression+='contains(.,'\''clean'\'') and '
       .../qa/workunits/ceph-helpers.sh:866: get_num_active_clean:  expression+='not(contains(.,'\''stale'\''))'
       .../qa/workunits/ceph-helpers.sh:867: get_num_active_clean:  expression+=')'
       .../qa/workunits/ceph-helpers.sh:872: get_num_active_clean:  xmlstarlet sel -t -m '//pg_stat/state[(contains(.,'\''active'\'') and contains(.,'\''clean'\'') and not(contains(.,'\''stale'\'')))]' -v . -n
       .../qa/workunits/ceph-helpers.sh:873: get_num_active_clean:  wc -l
       .../qa/workunits/ceph-helpers.sh:871: get_num_active_clean:  ceph --format xml pg dump pgs
       .../qa/workunits/ceph-helpers.sh:873: get_num_active_clean:  grep -v '^$'
       ../qa/workunits/ceph-helpers.sh:954: is_clean:  test 0 = 4
       ../qa/workunits/ceph-helpers.sh:954: is_clean:  return 1
       ../qa/workunits/ceph-helpers.sh:986: wait_for_clean:  get_is_making_recovery_progress
       .../qa/workunits/ceph-helpers.sh:839: get_is_making_recovery_progress:  ceph --format xml status
       .../qa/workunits/ceph-helpers.sh:839: get_is_making_recovery_progress:  xmlstarlet sel -t -m //pgmap/recovering_keys_per_sec -v . -o ' ' -t -m //pgmap/recovering_bytes_per_sec -v . -o ' ' -t -m //pgmap/recovering_objects_per_sec -v .
       ../qa/workunits/ceph-helpers.sh:839: get_is_making_recovery_progress:  local progress=
       ../qa/workunits/ceph-helpers.sh:840: get_is_making_recovery_progress:  test -n ''
       ../qa/workunits/ceph-helpers.sh:988: wait_for_clean:  ((  timer >= 120  ))
       .../qa/workunits/ceph-helpers.sh:993: wait_for_clean:  get_num_active_clean
       .../qa/workunits/ceph-helpers.sh:863: get_num_active_clean:  local 'expression=('
       .../qa/workunits/ceph-helpers.sh:864: get_num_active_clean:  expression+='contains(.,'\''active'\'') and '
       .../qa/workunits/ceph-helpers.sh:865: get_num_active_clean:  expression+='contains(.,'\''clean'\'') and '
       .../qa/workunits/ceph-helpers.sh:866: get_num_active_clean:  expression+='not(contains(.,'\''stale'\''))'
       .../qa/workunits/ceph-helpers.sh:867: get_num_active_clean:  expression+=')'
       .../qa/workunits/ceph-helpers.sh:871: get_num_active_clean:  ceph --format xml pg dump pgs
       .../qa/workunits/ceph-helpers.sh:872: get_num_active_clean:  xmlstarlet sel -t -m '//pg_stat/state[(contains(.,'\''active'\'') and contains(.,'\''clean'\'') and not(contains(.,'\''stale'\'')))]' -v . -n
       .../qa/workunits/ceph-helpers.sh:873: get_num_active_clean:  grep -v '^$'
       .../qa/workunits/ceph-helpers.sh:873: get_num_active_clean:  wc -l
       ../qa/workunits/ceph-helpers.sh:993: wait_for_clean:  cur_active_clean=0
       ../qa/workunits/ceph-helpers.sh:994: wait_for_clean:  test 0 '!=' 0
       ../qa/workunits/ceph-helpers.sh:998: wait_for_clean:  sleep 1
       .../qa/workunits/ceph-helpers.sh:999: wait_for_clean:  expr 118 + 1
       ../qa/workunits/ceph-helpers.sh:999: wait_for_clean:  timer=119
       ../qa/workunits/ceph-helpers.sh:985: wait_for_clean:  is_clean
       .../qa/workunits/ceph-helpers.sh:952: is_clean:  get_num_pgs
       .../qa/workunits/ceph-helpers.sh:898: get_num_pgs:  ceph --format xml status
       .../qa/workunits/ceph-helpers.sh:899: get_num_pgs:  xmlstarlet sel -t -m //pgmap/num_pgs -v .
       ../qa/workunits/ceph-helpers.sh:952: is_clean:  num_pgs=4
       ../qa/workunits/ceph-helpers.sh:953: is_clean:  test 4 '!=' 0
       .../qa/workunits/ceph-helpers.sh:954: is_clean:  get_num_active_clean
       .../qa/workunits/ceph-helpers.sh:863: get_num_active_clean:  local 'expression=('
       .../qa/workunits/ceph-helpers.sh:864: get_num_active_clean:  expression+='contains(.,'\''active'\'') and '
       .../qa/workunits/ceph-helpers.sh:865: get_num_active_clean:  expression+='contains(.,'\''clean'\'') and '
       .../qa/workunits/ceph-helpers.sh:866: get_num_active_clean:  expression+='not(contains(.,'\''stale'\''))'
       .../qa/workunits/ceph-helpers.sh:867: get_num_active_clean:  expression+=')'
       .../qa/workunits/ceph-helpers.sh:871: get_num_active_clean:  ceph --format xml pg dump pgs
       .../qa/workunits/ceph-helpers.sh:872: get_num_active_clean:  xmlstarlet sel -t -m '//pg_stat/state[(contains(.,'\''active'\'') and contains(.,'\''clean'\'') and not(contains(.,'\''stale'\'')))]' -v . -n
       .../qa/workunits/ceph-helpers.sh:873: get_num_active_clean:  grep -v '^$'
       .../qa/workunits/ceph-helpers.sh:873: get_num_active_clean:  wc -l
       ../qa/workunits/ceph-helpers.sh:954: is_clean:  test 0 = 4
       ../qa/workunits/ceph-helpers.sh:954: is_clean:  return 1
       ../qa/workunits/ceph-helpers.sh:986: wait_for_clean:  get_is_making_recovery_progress
       .../qa/workunits/ceph-helpers.sh:839: get_is_making_recovery_progress:  ceph --format xml status
       .../qa/workunits/ceph-helpers.sh:839: get_is_making_recovery_progress:  xmlstarlet sel -t -m //pgmap/recovering_keys_per_sec -v . -o ' ' -t -m //pgmap/recovering_bytes_per_sec -v . -o ' ' -t -m //pgmap/recovering_objects_per_sec -v .
       ../qa/workunits/ceph-helpers.sh:839: get_is_making_recovery_progress:  local progress=
       ../qa/workunits/ceph-helpers.sh:840: get_is_making_recovery_progress:  test -n ''
       ../qa/workunits/ceph-helpers.sh:988: wait_for_clean:  ((  timer >= 120  ))
       .../qa/workunits/ceph-helpers.sh:993: wait_for_clean:  get_num_active_clean
       .../qa/workunits/ceph-helpers.sh:863: get_num_active_clean:  local 'expression=('
       .../qa/workunits/ceph-helpers.sh:864: get_num_active_clean:  expression+='contains(.,'\''active'\'') and '
       .../qa/workunits/ceph-helpers.sh:865: get_num_active_clean:  expression+='contains(.,'\''clean'\'') and '
       .../qa/workunits/ceph-helpers.sh:866: get_num_active_clean:  expression+='not(contains(.,'\''stale'\''))'
       .../qa/workunits/ceph-helpers.sh:867: get_num_active_clean:  expression+=')'
       .../qa/workunits/ceph-helpers.sh:871: get_num_active_clean:  ceph --format xml pg dump pgs
       .../qa/workunits/ceph-helpers.sh:872: get_num_active_clean:  xmlstarlet sel -t -m '//pg_stat/state[(contains(.,'\''active'\'') and contains(.,'\''clean'\'') and not(contains(.,'\''stale'\'')))]' -v . -n
       .../qa/workunits/ceph-helpers.sh:873: get_num_active_clean:  grep -v '^$'
       .../qa/workunits/ceph-helpers.sh:873: get_num_active_clean:  wc -l
       ../qa/workunits/ceph-helpers.sh:993: wait_for_clean:  cur_active_clean=0
       ../qa/workunits/ceph-helpers.sh:994: wait_for_clean:  test 0 '!=' 0
       ../qa/workunits/ceph-helpers.sh:998: wait_for_clean:  sleep 1
       .../qa/workunits/ceph-helpers.sh:999: wait_for_clean:  expr 119 + 1
       ../qa/workunits/ceph-helpers.sh:999: wait_for_clean:  timer=120
       ../qa/workunits/ceph-helpers.sh:985: wait_for_clean:  is_clean
       .../qa/workunits/ceph-helpers.sh:952: is_clean:  get_num_pgs
       .../qa/workunits/ceph-helpers.sh:898: get_num_pgs:  ceph --format xml status
       .../qa/workunits/ceph-helpers.sh:899: get_num_pgs:  xmlstarlet sel -t -m //pgmap/num_pgs -v .
       ../qa/workunits/ceph-helpers.sh:952: is_clean:  num_pgs=4
       ../qa/workunits/ceph-helpers.sh:953: is_clean:  test 4 '!=' 0
       .../qa/workunits/ceph-helpers.sh:954: is_clean:  get_num_active_clean
       .../qa/workunits/ceph-helpers.sh:863: get_num_active_clean:  local 'expression=('
       .../qa/workunits/ceph-helpers.sh:864: get_num_active_clean:  expression+='contains(.,'\''active'\'') and '
       .../qa/workunits/ceph-helpers.sh:865: get_num_active_clean:  expression+='contains(.,'\''clean'\'') and '
       .../qa/workunits/ceph-helpers.sh:866: get_num_active_clean:  expression+='not(contains(.,'\''stale'\''))'
       .../qa/workunits/ceph-helpers.sh:867: get_num_active_clean:  expression+=')'
       .../qa/workunits/ceph-helpers.sh:871: get_num_active_clean:  ceph --format xml pg dump pgs
       .../qa/workunits/ceph-helpers.sh:872: get_num_active_clean:  xmlstarlet sel -t -m '//pg_stat/state[(contains(.,'\''active'\'') and contains(.,'\''clean'\'') and not(contains(.,'\''stale'\'')))]' -v . -n
       .../qa/workunits/ceph-helpers.sh:873: get_num_active_clean:  grep -v '^$'
       .../qa/workunits/ceph-helpers.sh:873: get_num_active_clean:  wc -l
       ../qa/workunits/ceph-helpers.sh:954: is_clean:  test 0 = 4
       ../qa/workunits/ceph-helpers.sh:954: is_clean:  return 1
       ../qa/workunits/ceph-helpers.sh:986: wait_for_clean:  get_is_making_recovery_progress
       .../qa/workunits/ceph-helpers.sh:839: get_is_making_recovery_progress:  ceph --format xml status
       .../qa/workunits/ceph-helpers.sh:839: get_is_making_recovery_progress:  xmlstarlet sel -t -m //pgmap/recovering_keys_per_sec -v . -o ' ' -t -m //pgmap/recovering_bytes_per_sec -v . -o ' ' -t -m //pgmap/recovering_objects_per_sec -v .
       ../qa/workunits/ceph-helpers.sh:839: get_is_making_recovery_progress:  local progress=
       ../qa/workunits/ceph-helpers.sh:840: get_is_making_recovery_progress:  test -n ''
       ../qa/workunits/ceph-helpers.sh:988: wait_for_clean:  ((  timer >= 120  ))
       ../qa/workunits/ceph-helpers.sh:989: wait_for_clean:  ceph report
       *** DEVELOPER MODE: setting PATH, PYTHONPATH and LD_LIBRARY_PATH ***
       report 249181440
+      {
           "cluster_fingerprint": "f71870a8-b77b-489c-bf68-762d3d294e6f",
           "version": "9.0.0-2049-g226d52b",
           "commit": "226d52b853411bda3e28634bef2f6f484bc48020",
           "timestamp": "2015-07-15 20:04:04.509438",
           "tag": "",
           "health": {
               "health": {
                   "health_services": [
+                      {
                           "mons": [
+                              {
                                   "name": "a",
                                   "kb_total": 232279476,
                                   "kb_used": 203099020,
                                   "kb_avail": 29164072,
                                   "avail_percent": 12,
                                   "last_updated": "2015-07-15 20:03:16.839547",
                                   "store_stats": {
                                       "bytes_total": 13632473,
                                       "bytes_sst": 603,
                                       "bytes_log": 13565952,
                                       "bytes_misc": 65918,
                                       "last_updated": "0.000000"
                                   },
                                   "health": "HEALTH_WARN",
                                   "health_detail": "low disk space"
+                              }
+                          ]
+                      }
+                  ]
               },
               "timechecks": {
                   "epoch": 2,
                   "round": 0,
                   "round_status": "finished"
               },
               "summary": [
+                  {
                       "severity": "HEALTH_WARN",
                       "summary": "4 pgs stuck inactive"
                   },
+                  {
                       "severity": "HEALTH_WARN",
                       "summary": "4 pgs stuck unclean"
                   },
+                  {
                       "severity": "HEALTH_WARN",
                       "summary": "too few PGs per OSD (4 < min 30)"
                   },
+                  {
                       "severity": "HEALTH_WARN",
                       "summary": "1\/1 in osds are down"
                   },
+                  {
                       "severity": "HEALTH_WARN",
                       "summary": "mon.a low disk space"
+                  }
               ],
               "overall_status": "HEALTH_WARN",
               "detail": []
           },
           "monmap_first_committed": 1,
           "monmap_last_committed": 1,
           "monmap": {
               "epoch": 1,
               "fsid": "3f1b7cce-0c19-4c66-990c-9bbd1a39c039",
               "modified": "0.000000",
               "created": "0.000000",
               "mons": [
+                  {
                       "rank": 0,
                       "name": "a",
                       "addr": "127.0.0.1:7109\/0"
+                  }
+              ]
           },
           "quorum": [
           ],
           "osdmap": {
               "epoch": 8,
               "fsid": "3f1b7cce-0c19-4c66-990c-9bbd1a39c039",
               "created": "2015-07-15 19:59:16.864574",
               "modified": "2015-07-15 19:59:21.946332",
               "flags": "",
               "cluster_snapshot": "",
               "pool_max": 1,
               "max_osd": 1,
               "pools": [
+                  {
                       "pool": 1,
                       "pool_name": "rbd",
                       "flags": 1,
                       "flags_names": "hashpspool",
                       "type": 1,
                       "size": 1,
                       "min_size": 1,
                       "crush_ruleset": 0,
                       "object_hash": 2,
                       "pg_num": 4,
                       "pg_placement_num": 4,
                       "crash_replay_interval": 0,
                       "last_change": "3",
                       "last_force_op_resend": "0",
                       "auid": 0,
                       "snap_mode": "selfmanaged",
                       "snap_seq": 0,
                       "snap_epoch": 0,
                       "pool_snaps": [],
                       "removed_snaps": "[]",
                       "quota_max_bytes": 0,
                       "quota_max_objects": 0,
                       "tiers": [],
                       "tier_of": -1,
                       "read_tier": -1,
                       "write_tier": -1,
                       "cache_mode": "none",
                       "target_max_bytes": 0,
                       "target_max_objects": 0,
                       "cache_target_dirty_ratio_micro": 400000,
                       "cache_target_dirty_high_ratio_micro": 600000,
                       "cache_target_full_ratio_micro": 800000,
                       "cache_min_flush_age": 0,
                       "cache_min_evict_age": 0,
                       "erasure_code_profile": "",
                       "hit_set_params": {
                           "type": "none"
                       },
                       "hit_set_period": 0,
                       "hit_set_count": 0,
                       "min_read_recency_for_promote": 0,
                       "stripe_width": 0,
                       "expected_num_objects": 0
+                  }
               ],
               "osds": [
+                  {
                       "osd": 0,
                       "uuid": "d0bf3a6f-f6c7-47d6-be01-3b66da6c1aed",
                       "up": 0,
                       "in": 1,
                       "weight": 1.000000,
                       "primary_affinity": 1.000000,
                       "last_clean_begin": 0,
                       "last_clean_end": 0,
                       "up_from": 5,
                       "up_thru": 6,
                       "down_at": 8,
                       "lost_at": 0,
                       "public_addr": "127.0.0.1:6804\/29933",
                       "cluster_addr": "127.0.0.1:6805\/29933",
                       "heartbeat_back_addr": "127.0.0.1:6806\/29933",
                       "heartbeat_front_addr": "127.0.0.1:6807\/29933",
                       "state": [
                           "exists"
+                      ]
+                  }
               ],
               "osd_xinfo": [
+                  {
                       "osd": 0,
                       "down_stamp": "2015-07-15 19:59:21.946332",
                       "laggy_probability": 0.000000,
                       "laggy_interval": 0,
                       "features": 2251799813685247,
                       "old_weight": 0
+                  }
               ],
               "pg_temp": [],
               "primary_temp": [],
               "blacklist": {},
               "erasure_code_profiles": {
                   "default": {
                       "directory": ".libs",
                       "k": "2",
                       "m": "1",
                       "plugin": "jerasure",
                       "technique": "reed_sol_van"
+                  }
+              }
           },
           "osd_metadata": [
+              {
                   "id": 0,
                   "arch": "x86_64",
                   "back_addr": "127.0.0.1:6805\/29933",
                   "backend_filestore_dev_node": "unknown",
                   "backend_filestore_partition_path": "unknown",
                   "ceph_version": "ceph version 9.0.0-2049-g226d52b (226d52b853411bda3e28634bef2f6f484bc48020)",
                   "cpu": "Intel(R) Xeon(R) CPU E5-2620 v2 @ 2.10GHz",
                   "distro": "CentOS",
                   "distro_codename": "Core",
                   "distro_description": "CentOS Linux release 7.0.1406 (Core) ",
                   "distro_version": "7.0.1406",
                   "filestore_backend": "generic",
                   "filestore_f_type": "0xef53",
                   "front_addr": "127.0.0.1:6804\/29933",
                   "hb_back_addr": "127.0.0.1:6806\/29933",
                   "hb_front_addr": "127.0.0.1:6807\/29933",
                   "hostname": "3933a54df967",
                   "kernel_description": "#1 SMP Fri Nov 21 23:26:41 UTC 2014",
                   "kernel_version": "3.17.4-200.fc20.x86_64",
                   "mem_swap_kb": "0",
                   "mem_total_kb": "57683664",
                   "os": "Linux",
                   "osd_data": "testdir\/ceph-helpers\/0",
                   "osd_journal": "testdir\/ceph-helpers\/0\/journal",
                   "osd_objectstore": "filestore"
+              }
           ],
           "osdmap_first_committed": 1,
           "osdmap_last_committed": 8,
           "crushmap": {
               "devices": [
+                  {
                       "id": 0,
                       "name": "osd.0"
+                  }
               ],
               "types": [
+                  {
                       "type_id": 0,
                       "name": "osd"
                   },
+                  {
                       "type_id": 1,
                       "name": "host"
                   },
+                  {
                       "type_id": 2,
                       "name": "chassis"
                   },
+                  {
                       "type_id": 3,
                       "name": "rack"
                   },
+                  {
                       "type_id": 4,
                       "name": "row"
                   },
+                  {
                       "type_id": 5,
                       "name": "pdu"
                   },
+                  {
                       "type_id": 6,
                       "name": "pod"
                   },
+                  {
                       "type_id": 7,
                       "name": "room"
                   },
+                  {
                       "type_id": 8,
                       "name": "datacenter"
                   },
+                  {
                       "type_id": 9,
                       "name": "region"
                   },
+                  {
                       "type_id": 10,
                       "name": "root"
+                  }
               ],
               "buckets": [
+                  {
                       "id": -1,
                       "name": "default",
                       "type_id": 10,
                       "type_name": "root",
                       "weight": 65536,
                       "alg": "straw",
                       "hash": "rjenkins1",
                       "items": [
+                          {
                               "id": -2,
                               "weight": 65536,
                               "pos": 0
+                          }
+                      ]
                   },
+                  {
                       "id": -2,
                       "name": "localhost",
                       "type_id": 1,
                       "type_name": "host",
                       "weight": 65536,
                       "alg": "straw",
                       "hash": "rjenkins1",
                       "items": [
+                          {
                               "id": 0,
                               "weight": 65536,
                               "pos": 0
+                          }
+                      ]
+                  }
               ],
               "rules": [
+                  {
                       "rule_id": 0,
                       "rule_name": "replicated_ruleset",
                       "ruleset": 0,
                       "type": 1,
                       "min_size": 1,
                       "max_size": 10,
                       "steps": [
+                          {
                               "op": "take",
                               "item": -1,
                               "item_name": "default"
                           },
+                          {
                               "op": "choose_firstn",
                               "num": 0,
                               "type": "osd"
                           },
+                          {
                               "op": "emit"
+                          }
+                      ]
+                  }
               ],
               "tunables": {
                   "choose_local_tries": 0,
                   "choose_local_fallback_tries": 0,
                   "choose_total_tries": 50,
                   "chooseleaf_descend_once": 1,
                   "chooseleaf_vary_r": 0,
                   "straw_calc_version": 1,
                   "allowed_bucket_algs": 22,
                   "profile": "unknown",
                   "optimal_tunables": 0,
                   "legacy_tunables": 0,
                   "require_feature_tunables": 1,
                   "require_feature_tunables2": 1,
                   "require_feature_tunables3": 0,
                   "has_v2_rules": 0,
                   "has_v3_rules": 0,
                   "has_v4_buckets": 0
+              }
           },
           "mdsmap": {
               "epoch": 1,
               "flags": 0,
               "created": "0.000000",
               "modified": "2015-07-15 19:59:16.864159",
               "tableserver": 0,
               "root": 0,
               "session_timeout": 0,
               "session_autoclose": 0,
               "max_file_size": 0,
               "last_failure": 0,
               "last_failure_osd_epoch": 0,
               "compat": {
                   "compat": {},
                   "ro_compat": {},
                   "incompat": {}
               },
               "max_mds": 0,
               "in": [],
               "up": {},
               "failed": [],
               "damaged": [],
               "stopped": [],
               "info": {},
               "data_pools": [],
               "metadata_pool": 0,
               "enabled": false,
               "fs_name": "cephfs"
           },
           "mdsmap_first_committed": 1,
           "mdsmap_last_committed": 1,
           "pgmap": {
               "version": 9,
               "stamp": "2015-07-15 19:59:21.949132",
               "last_osdmap_epoch": 8,
               "last_pg_scan": 3,
               "full_ratio": 0.990000,
               "near_full_ratio": 0.850000,
               "pg_stats_sum": {
                   "stat_sum": {
                       "num_bytes": 0,
                       "num_objects": 0,
                       "num_object_clones": 0,
                       "num_object_copies": 0,
                       "num_objects_missing_on_primary": 0,
                       "num_objects_degraded": 0,
                       "num_objects_misplaced": 0,
                       "num_objects_unfound": 0,
                       "num_objects_dirty": 0,
                       "num_whiteouts": 0,
                       "num_read": 0,
                       "num_read_kb": 0,
                       "num_write": 0,
                       "num_write_kb": 0,
                       "num_scrub_errors": 0,
                       "num_shallow_scrub_errors": 0,
                       "num_deep_scrub_errors": 0,
                       "num_objects_recovered": 0,
                       "num_bytes_recovered": 0,
                       "num_keys_recovered": 0,
                       "num_objects_omap": 0,
                       "num_objects_hit_set_archive": 0,
                       "num_bytes_hit_set_archive": 0
                   },
                   "log_size": 0,
                   "ondisk_log_size": 0,
                   "up": 0,
                   "acting": 0
               },
               "osd_stats_sum": {
                   "kb": 0,
                   "kb_used": 0,
                   "kb_avail": 0,
                   "hb_in": [],
                   "hb_out": [],
                   "snap_trim_queue_len": 0,
                   "num_snap_trimming": 0,
                   "op_queue_age_hist": {
                       "histogram": [],
                       "upper_bound": 1
                   },
                   "fs_perf_stat": {
                       "commit_latency_ms": 0,
                       "apply_latency_ms": 0
+                  }
               },
               "pg_stats_delta": {
                   "stat_sum": {
                       "num_bytes": 0,
                       "num_objects": 0,
                       "num_object_clones": 0,
                       "num_object_copies": 0,
                       "num_objects_missing_on_primary": 0,
                       "num_objects_degraded": 0,
                       "num_objects_misplaced": 0,
                       "num_objects_unfound": 0,
                       "num_objects_dirty": 0,
                       "num_whiteouts": 0,
                       "num_read": 0,
                       "num_read_kb": 0,
                       "num_write": 0,
                       "num_write_kb": 0,
                       "num_scrub_errors": 0,
                       "num_shallow_scrub_errors": 0,
                       "num_deep_scrub_errors": 0,
                       "num_objects_recovered": 0,
                       "num_bytes_recovered": 0,
                       "num_keys_recovered": 0,
                       "num_objects_omap": 0,
                       "num_objects_hit_set_archive": 0,
                       "num_bytes_hit_set_archive": 0
                   },
                   "log_size": 0,
                   "ondisk_log_size": 0,
                   "up": 0,
                   "acting": 0
               },
               "pg_stats": [
+                  {
                       "pgid": "1.2",
                       "version": "0'0",
                       "reported_seq": "0",
                       "reported_epoch": "0",
                       "state": "creating",
                       "last_fresh": "0.000000",
                       "last_change": "0.000000",
                       "last_active": "0.000000",
                       "last_peered": "0.000000",
                       "last_clean": "0.000000",
                       "last_became_active": "0.000000",
                       "last_became_peered": "0.000000",
                       "last_unstale": "0.000000",
                       "last_undegraded": "0.000000",
                       "last_fullsized": "0.000000",
                       "mapping_epoch": 0,
                       "log_start": "0'0",
                       "ondisk_log_start": "0'0",
                       "created": 3,
                       "last_epoch_clean": 0,
                       "parent": "0.0",
                       "parent_split_bits": 0,
                       "last_scrub": "0'0",
                       "last_scrub_stamp": "2015-07-15 19:59:17.874137",
                       "last_deep_scrub": "0'0",
                       "last_deep_scrub_stamp": "2015-07-15 19:59:17.874137",
                       "last_clean_scrub_stamp": "2015-07-15 19:59:17.874137",
                       "log_size": 0,
                       "ondisk_log_size": 0,
                       "stats_invalid": "0",
                       "stat_sum": {
                           "num_bytes": 0,
                           "num_objects": 0,
                           "num_object_clones": 0,
                           "num_object_copies": 0,
                           "num_objects_missing_on_primary": 0,
                           "num_objects_degraded": 0,
                           "num_objects_misplaced": 0,
                           "num_objects_unfound": 0,
                           "num_objects_dirty": 0,
                           "num_whiteouts": 0,
                           "num_read": 0,
                           "num_read_kb": 0,
                           "num_write": 0,
                           "num_write_kb": 0,
                           "num_scrub_errors": 0,
                           "num_shallow_scrub_errors": 0,
                           "num_deep_scrub_errors": 0,
                           "num_objects_recovered": 0,
                           "num_bytes_recovered": 0,
                           "num_keys_recovered": 0,
                           "num_objects_omap": 0,
                           "num_objects_hit_set_archive": 0,
                           "num_bytes_hit_set_archive": 0
                       },
                       "up": [],
                       "acting": [],
                       "blocked_by": [],
                       "up_primary": -1,
                       "acting_primary": -1
                   },
+                  {
                       "pgid": "1.3",
                       "version": "0'0",
                       "reported_seq": "0",
                       "reported_epoch": "0",
                       "state": "creating",
                       "last_fresh": "0.000000",
                       "last_change": "0.000000",
                       "last_active": "0.000000",
                       "last_peered": "0.000000",
                       "last_clean": "0.000000",
                       "last_became_active": "0.000000",
                       "last_became_peered": "0.000000",
                       "last_unstale": "0.000000",
                       "last_undegraded": "0.000000",
                       "last_fullsized": "0.000000",
                       "mapping_epoch": 0,
                       "log_start": "0'0",
                       "ondisk_log_start": "0'0",
                       "created": 3,
                       "last_epoch_clean": 0,
                       "parent": "0.0",
                       "parent_split_bits": 0,
                       "last_scrub": "0'0",
                       "last_scrub_stamp": "2015-07-15 19:59:17.874144",
                       "last_deep_scrub": "0'0",
                       "last_deep_scrub_stamp": "2015-07-15 19:59:17.874144",
                       "last_clean_scrub_stamp": "2015-07-15 19:59:17.874144",
                       "log_size": 0,
                       "ondisk_log_size": 0,
                       "stats_invalid": "0",
                       "stat_sum": {
                           "num_bytes": 0,
                           "num_objects": 0,
                           "num_object_clones": 0,
                           "num_object_copies": 0,
                           "num_objects_missing_on_primary": 0,
                           "num_objects_degraded": 0,
                           "num_objects_misplaced": 0,
                           "num_objects_unfound": 0,
                           "num_objects_dirty": 0,
                           "num_whiteouts": 0,
                           "num_read": 0,
                           "num_read_kb": 0,
                           "num_write": 0,
                           "num_write_kb": 0,
                           "num_scrub_errors": 0,
                           "num_shallow_scrub_errors": 0,
                           "num_deep_scrub_errors": 0,
                           "num_objects_recovered": 0,
                           "num_bytes_recovered": 0,
                           "num_keys_recovered": 0,
                           "num_objects_omap": 0,
                           "num_objects_hit_set_archive": 0,
                           "num_bytes_hit_set_archive": 0
                       },
                       "up": [],
                       "acting": [],
                       "blocked_by": [],
                       "up_primary": -1,
                       "acting_primary": -1
                   },
+                  {
                       "pgid": "1.0",
                       "version": "0'0",
                       "reported_seq": "0",
                       "reported_epoch": "0",
                       "state": "creating",
                       "last_fresh": "0.000000",
                       "last_change": "0.000000",
                       "last_active": "0.000000",
                       "last_peered": "0.000000",
                       "last_clean": "0.000000",
                       "last_became_active": "0.000000",
                       "last_became_peered": "0.000000",
                       "last_unstale": "0.000000",
                       "last_undegraded": "0.000000",
                       "last_fullsized": "0.000000",
                       "mapping_epoch": 0,
                       "log_start": "0'0",
                       "ondisk_log_start": "0'0",
                       "created": 3,
                       "last_epoch_clean": 0,
                       "parent": "0.0",
                       "parent_split_bits": 0,
                       "last_scrub": "0'0",
                       "last_scrub_stamp": "2015-07-15 19:59:17.874125",
                       "last_deep_scrub": "0'0",
                       "last_deep_scrub_stamp": "2015-07-15 19:59:17.874125",
                       "last_clean_scrub_stamp": "2015-07-15 19:59:17.874125",
                       "log_size": 0,
                       "ondisk_log_size": 0,
                       "stats_invalid": "0",
                       "stat_sum": {
                           "num_bytes": 0,
                           "num_objects": 0,
                           "num_object_clones": 0,
                           "num_object_copies": 0,
                           "num_objects_missing_on_primary": 0,
                           "num_objects_degraded": 0,
                           "num_objects_misplaced": 0,
                           "num_objects_unfound": 0,
                           "num_objects_dirty": 0,
                           "num_whiteouts": 0,
                           "num_read": 0,
                           "num_read_kb": 0,
                           "num_write": 0,
                           "num_write_kb": 0,
                           "num_scrub_errors": 0,
                           "num_shallow_scrub_errors": 0,
                           "num_deep_scrub_errors": 0,
                           "num_objects_recovered": 0,
                           "num_bytes_recovered": 0,
                           "num_keys_recovered": 0,
                           "num_objects_omap": 0,
                           "num_objects_hit_set_archive": 0,
                           "num_bytes_hit_set_archive": 0
                       },
                       "up": [],
                       "acting": [],
                       "blocked_by": [],
                       "up_primary": -1,
                       "acting_primary": -1
                   },
+                  {
                       "pgid": "1.1",
                       "version": "0'0",
                       "reported_seq": "0",
                       "reported_epoch": "0",
                       "state": "creating",
                       "last_fresh": "0.000000",
                       "last_change": "0.000000",
                       "last_active": "0.000000",
                       "last_peered": "0.000000",
                       "last_clean": "0.000000",
                       "last_became_active": "0.000000",
                       "last_became_peered": "0.000000",
                       "last_unstale": "0.000000",
                       "last_undegraded": "0.000000",
                       "last_fullsized": "0.000000",
                       "mapping_epoch": 0,
                       "log_start": "0'0",
                       "ondisk_log_start": "0'0",
                       "created": 3,
                       "last_epoch_clean": 0,
                       "parent": "0.0",
                       "parent_split_bits": 0,
                       "last_scrub": "0'0",
                       "last_scrub_stamp": "2015-07-15 19:59:17.874132",
                       "last_deep_scrub": "0'0",
                       "last_deep_scrub_stamp": "2015-07-15 19:59:17.874132",
                       "last_clean_scrub_stamp": "2015-07-15 19:59:17.874132",
                       "log_size": 0,
                       "ondisk_log_size": 0,
                       "stats_invalid": "0",
                       "stat_sum": {
                           "num_bytes": 0,
                           "num_objects": 0,
                           "num_object_clones": 0,
                           "num_object_copies": 0,
                           "num_objects_missing_on_primary": 0,
                           "num_objects_degraded": 0,
                           "num_objects_misplaced": 0,
                           "num_objects_unfound": 0,
                           "num_objects_dirty": 0,
                           "num_whiteouts": 0,
                           "num_read": 0,
                           "num_read_kb": 0,
                           "num_write": 0,
                           "num_write_kb": 0,
                           "num_scrub_errors": 0,
                           "num_shallow_scrub_errors": 0,
                           "num_deep_scrub_errors": 0,
                           "num_objects_recovered": 0,
                           "num_bytes_recovered": 0,
                           "num_keys_recovered": 0,
                           "num_objects_omap": 0,
                           "num_objects_hit_set_archive": 0,
                           "num_bytes_hit_set_archive": 0
                       },
                       "up": [],
                       "acting": [],
                       "blocked_by": [],
                       "up_primary": -1,
                       "acting_primary": -1
+                  }
               ],
               "pool_stats": [
+                  {
                       "poolid": 1,
                       "stat_sum": {
                           "num_bytes": 0,
                           "num_objects": 0,
                           "num_object_clones": 0,
                           "num_object_copies": 0,
                           "num_objects_missing_on_primary": 0,
                           "num_objects_degraded": 0,
                           "num_objects_misplaced": 0,
                           "num_objects_unfound": 0,
                           "num_objects_dirty": 0,
                           "num_whiteouts": 0,
                           "num_read": 0,
                           "num_read_kb": 0,
                           "num_write": 0,
                           "num_write_kb": 0,
                           "num_scrub_errors": 0,
                           "num_shallow_scrub_errors": 0,
                           "num_deep_scrub_errors": 0,
                           "num_objects_recovered": 0,
                           "num_bytes_recovered": 0,
                           "num_keys_recovered": 0,
                           "num_objects_omap": 0,
                           "num_objects_hit_set_archive": 0,
                           "num_bytes_hit_set_archive": 0
                       },
                       "log_size": 0,
                       "ondisk_log_size": 0,
                       "up": 0,
                       "acting": 0
+                  }
               ],
               "osd_stats": []
           },
           "pgmap_first_committed": 1,
           "pgmap_last_committed": 9,
           "auth": {
               "first_committed": 1,
               "last_committed": 3,
               "num_secrets": 1
           },
           "paxos": {
               "first_committed": 1,
               "last_committed": 519,
               "last_pn": 0,
               "accepted_pn": 0
+          }
+      }
       ../qa/workunits/ceph-helpers.sh:990: wait_for_clean:  return 1
       ../qa/workunits/ceph-helpers.sh:937: test_get_last_scrub_stamp:  return 1
       ../qa/workunits/ceph-helpers.sh:1199: run_tests:  return 1

(1-1/1)

Project

General

Profile

Ceph

Bug #12340 » test.txt