+ PS4='${BASH_SOURCE[0]}:$LINENO: ${FUNCNAME[0]}: ' ../qa/workunits/ceph-helpers.sh:1188: run_tests: export PATH=:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin ../qa/workunits/ceph-helpers.sh:1188: run_tests: PATH=:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin ../qa/workunits/ceph-helpers.sh:1189: run_tests: export CEPH_MON=127.0.0.1:7109 ../qa/workunits/ceph-helpers.sh:1189: run_tests: CEPH_MON=127.0.0.1:7109 ../qa/workunits/ceph-helpers.sh:1190: run_tests: export CEPH_ARGS .../qa/workunits/ceph-helpers.sh:1191: run_tests: uuidgen ../qa/workunits/ceph-helpers.sh:1191: run_tests: CEPH_ARGS+='--fsid=3f1b7cce-0c19-4c66-990c-9bbd1a39c039 --auth-supported=none ' ../qa/workunits/ceph-helpers.sh:1192: run_tests: CEPH_ARGS+='--mon-host=127.0.0.1:7109 ' ../qa/workunits/ceph-helpers.sh:1193: run_tests: export CEPH_CONF=/dev/null ../qa/workunits/ceph-helpers.sh:1193: run_tests: CEPH_CONF=/dev/null .../qa/workunits/ceph-helpers.sh:1195: run_tests: set .../qa/workunits/ceph-helpers.sh:1195: run_tests: sed -n -e 's/^\(test_[0-9a-z_]*\) .*/\1/p' ../qa/workunits/ceph-helpers.sh:1195: run_tests: local 'funcs=test_activate_osd test_erasure_code_plugin_exists test_expect_failure test_get_config test_get_is_making_recovery_progress test_get_last_scrub_stamp test_get_not_primary test_get_num_active_clean test_get_num_pgs test_get_osds test_get_pg test_get_primary test_is_clean test_kill_daemons test_objectstore_tool test_repair test_run_mon test_run_osd test_set_config test_setup test_teardown test_wait_for_clean test_wait_for_osd' ../qa/workunits/ceph-helpers.sh:1196: run_tests: local dir=testdir/ceph-helpers ../qa/workunits/ceph-helpers.sh:1198: run_tests: for func in '$funcs' ../qa/workunits/ceph-helpers.sh:1199: run_tests: test_activate_osd testdir/ceph-helpers ../qa/workunits/ceph-helpers.sh:518: test_activate_osd: local dir=testdir/ceph-helpers ../qa/workunits/ceph-helpers.sh:520: test_activate_osd: setup testdir/ceph-helpers ../qa/workunits/ceph-helpers.sh:92: setup: local dir=testdir/ceph-helpers ../qa/workunits/ceph-helpers.sh:93: setup: teardown testdir/ceph-helpers ../qa/workunits/ceph-helpers.sh:117: teardown: local dir=testdir/ceph-helpers ../qa/workunits/ceph-helpers.sh:118: teardown: kill_daemons testdir/ceph-helpers .../qa/workunits/ceph-helpers.sh:192: kill_daemons: shopt -q -o xtrace .../qa/workunits/ceph-helpers.sh:192: kill_daemons: echo true ../qa/workunits/ceph-helpers.sh:192: kill_daemons: local trace=true ../qa/workunits/ceph-helpers.sh:193: kill_daemons: true ../qa/workunits/ceph-helpers.sh:193: kill_daemons: shopt -u -o xtrace ../qa/workunits/ceph-helpers.sh:219: kill_daemons: return 0 .../qa/workunits/ceph-helpers.sh:119: teardown: stat -f -c %T . ../qa/workunits/ceph-helpers.sh:119: teardown: '[' ext2/ext3 == btrfs ']' ../qa/workunits/ceph-helpers.sh:122: teardown: rm -fr testdir/ceph-helpers ../qa/workunits/ceph-helpers.sh:94: setup: mkdir -p testdir/ceph-helpers ../qa/workunits/ceph-helpers.sh:522: test_activate_osd: run_mon testdir/ceph-helpers a ../qa/workunits/ceph-helpers.sh:282: run_mon: local dir=testdir/ceph-helpers ../qa/workunits/ceph-helpers.sh:283: run_mon: shift ../qa/workunits/ceph-helpers.sh:284: run_mon: local id=a ../qa/workunits/ceph-helpers.sh:285: run_mon: shift ../qa/workunits/ceph-helpers.sh:286: run_mon: local data=testdir/ceph-helpers/a ../qa/workunits/ceph-helpers.sh:289: run_mon: ceph-mon --id a --mkfs --mon-data=testdir/ceph-helpers/a --run-dir=testdir/ceph-helpers ceph-mon: mon.noname-a 127.0.0.1:7109/0 is local, renaming to mon.a ceph-mon: set fsid to 3f1b7cce-0c19-4c66-990c-9bbd1a39c039 ceph-mon: created monfs at testdir/ceph-helpers/a for mon.a ../qa/workunits/ceph-helpers.sh:296: run_mon: ceph-mon --id a --mon-osd-full-ratio=.99 --mon-data-avail-crit=1 --paxos-propose-interval=0.1 --osd-crush-chooseleaf-type=0 --osd-pool-default-erasure-code-directory=.libs --debug-mon 20 --debug-ms 20 --debug-paxos 20 --chdir= --mon-data=testdir/ceph-helpers/a '--log-file=testdir/ceph-helpers/$name.log' '--admin-socket=testdir/ceph-helpers/$cluster-$name.asok' --mon-cluster-log-file=testdir/ceph-helpers/log --run-dir=testdir/ceph-helpers '--pid-file=testdir/ceph-helpers/$name.pid' ../qa/workunits/ceph-helpers.sh:314: run_mon: cat .../qa/workunits/ceph-helpers.sh:314: run_mon: get_config mon a fsid .../qa/workunits/ceph-helpers.sh:653: get_config: local daemon=mon .../qa/workunits/ceph-helpers.sh:654: get_config: local id=a .../qa/workunits/ceph-helpers.sh:655: get_config: local config=fsid .../qa/workunits/ceph-helpers.sh:657: get_config: CEPH_ARGS= .../qa/workunits/ceph-helpers.sh:657: get_config: ceph --format xml daemon testdir/ceph-helpers/ceph-mon.a.asok config get fsid .../qa/workunits/ceph-helpers.sh:660: get_config: xmlstarlet sel -t -m //fsid -v . -n .../qa/workunits/ceph-helpers.sh:314: run_mon: get_config mon a mon_host .../qa/workunits/ceph-helpers.sh:653: get_config: local daemon=mon .../qa/workunits/ceph-helpers.sh:654: get_config: local id=a .../qa/workunits/ceph-helpers.sh:655: get_config: local config=mon_host .../qa/workunits/ceph-helpers.sh:660: get_config: xmlstarlet sel -t -m //mon_host -v . -n .../qa/workunits/ceph-helpers.sh:657: get_config: CEPH_ARGS= .../qa/workunits/ceph-helpers.sh:657: get_config: ceph --format xml daemon testdir/ceph-helpers/ceph-mon.a.asok config get mon_host .../qa/workunits/ceph-helpers.sh:319: run_mon: get_config mon a mon_initial_members .../qa/workunits/ceph-helpers.sh:653: get_config: local daemon=mon .../qa/workunits/ceph-helpers.sh:654: get_config: local id=a .../qa/workunits/ceph-helpers.sh:655: get_config: local config=mon_initial_members .../qa/workunits/ceph-helpers.sh:657: get_config: CEPH_ARGS= .../qa/workunits/ceph-helpers.sh:657: get_config: ceph --format xml daemon testdir/ceph-helpers/ceph-mon.a.asok config get mon_initial_members .../qa/workunits/ceph-helpers.sh:660: get_config: xmlstarlet sel -t -m //mon_initial_members -v . -n ../qa/workunits/ceph-helpers.sh:319: run_mon: test -z '' ../qa/workunits/ceph-helpers.sh:320: run_mon: ceph osd pool delete rbd rbd --yes-i-really-really-mean-it *** DEVELOPER MODE: setting PATH, PYTHONPATH and LD_LIBRARY_PATH *** pool 'rbd' removed ../qa/workunits/ceph-helpers.sh:321: run_mon: ceph osd pool create rbd 4 *** DEVELOPER MODE: setting PATH, PYTHONPATH and LD_LIBRARY_PATH *** pool 'rbd' created ../qa/workunits/ceph-helpers.sh:524: test_activate_osd: run_osd testdir/ceph-helpers 0 ../qa/workunits/ceph-helpers.sh:397: run_osd: local dir=testdir/ceph-helpers ../qa/workunits/ceph-helpers.sh:398: run_osd: shift ../qa/workunits/ceph-helpers.sh:399: run_osd: local id=0 ../qa/workunits/ceph-helpers.sh:400: run_osd: shift ../qa/workunits/ceph-helpers.sh:401: run_osd: local osd_data=testdir/ceph-helpers/0 ../qa/workunits/ceph-helpers.sh:403: run_osd: local ceph_disk_args ../qa/workunits/ceph-helpers.sh:404: run_osd: ceph_disk_args+=' --statedir=testdir/ceph-helpers' ../qa/workunits/ceph-helpers.sh:405: run_osd: ceph_disk_args+=' --sysconfdir=testdir/ceph-helpers' ../qa/workunits/ceph-helpers.sh:406: run_osd: ceph_disk_args+=' --prepend-to-path=' ../qa/workunits/ceph-helpers.sh:408: run_osd: mkdir -p testdir/ceph-helpers/0 ../qa/workunits/ceph-helpers.sh:409: run_osd: ceph-disk --statedir=testdir/ceph-helpers --sysconfdir=testdir/ceph-helpers --prepend-to-path= prepare testdir/ceph-helpers/0 2015-07-15 19:58:41.930854 7f37c836c7c0 -1 warning: unable to create /var/run/ceph: (13) Permission denied 2015-07-15 19:58:41.944397 7fc0874367c0 -1 warning: unable to create /var/run/ceph: (13) Permission denied 2015-07-15 19:58:41.960148 7f2d771be7c0 -1 warning: unable to create /var/run/ceph: (13) Permission denied ../qa/workunits/ceph-helpers.sh:412: run_osd: activate_osd testdir/ceph-helpers 0 ../qa/workunits/ceph-helpers.sh:479: activate_osd: local dir=testdir/ceph-helpers ../qa/workunits/ceph-helpers.sh:480: activate_osd: shift ../qa/workunits/ceph-helpers.sh:481: activate_osd: local id=0 ../qa/workunits/ceph-helpers.sh:482: activate_osd: shift ../qa/workunits/ceph-helpers.sh:483: activate_osd: local osd_data=testdir/ceph-helpers/0 ../qa/workunits/ceph-helpers.sh:485: activate_osd: local ceph_disk_args ../qa/workunits/ceph-helpers.sh:486: activate_osd: ceph_disk_args+=' --statedir=testdir/ceph-helpers' ../qa/workunits/ceph-helpers.sh:487: activate_osd: ceph_disk_args+=' --sysconfdir=testdir/ceph-helpers' ../qa/workunits/ceph-helpers.sh:488: activate_osd: ceph_disk_args+=' --prepend-to-path=' ../qa/workunits/ceph-helpers.sh:490: activate_osd: local 'ceph_args=--fsid=3f1b7cce-0c19-4c66-990c-9bbd1a39c039 --auth-supported=none --mon-host=127.0.0.1:7109 ' ../qa/workunits/ceph-helpers.sh:491: activate_osd: ceph_args+=' --osd-backfill-full-ratio=.99' ../qa/workunits/ceph-helpers.sh:492: activate_osd: ceph_args+=' --osd-failsafe-full-ratio=.99' ../qa/workunits/ceph-helpers.sh:493: activate_osd: ceph_args+=' --osd-journal-size=100' ../qa/workunits/ceph-helpers.sh:494: activate_osd: ceph_args+=' --osd-data=testdir/ceph-helpers/0' ../qa/workunits/ceph-helpers.sh:495: activate_osd: ceph_args+=' --chdir=' ../qa/workunits/ceph-helpers.sh:496: activate_osd: ceph_args+=' --osd-pool-default-erasure-code-directory=.libs' ../qa/workunits/ceph-helpers.sh:497: activate_osd: ceph_args+=' --osd-class-dir=.libs' ../qa/workunits/ceph-helpers.sh:498: activate_osd: ceph_args+=' --run-dir=testdir/ceph-helpers' ../qa/workunits/ceph-helpers.sh:499: activate_osd: ceph_args+=' --debug-osd=20' ../qa/workunits/ceph-helpers.sh:500: activate_osd: ceph_args+=' --log-file=testdir/ceph-helpers/$name.log' ../qa/workunits/ceph-helpers.sh:501: activate_osd: ceph_args+=' --pid-file=testdir/ceph-helpers/$name.pid' ../qa/workunits/ceph-helpers.sh:502: activate_osd: ceph_args+=' ' ../qa/workunits/ceph-helpers.sh:503: activate_osd: ceph_args+= ../qa/workunits/ceph-helpers.sh:504: activate_osd: mkdir -p testdir/ceph-helpers/0 ../qa/workunits/ceph-helpers.sh:505: activate_osd: CEPH_ARGS='--fsid=3f1b7cce-0c19-4c66-990c-9bbd1a39c039 --auth-supported=none --mon-host=127.0.0.1:7109 --osd-backfill-full-ratio=.99 --osd-failsafe-full-ratio=.99 --osd-journal-size=100 --osd-data=testdir/ceph-helpers/0 --chdir= --osd-pool-default-erasure-code-directory=.libs --osd-class-dir=.libs --run-dir=testdir/ceph-helpers --debug-osd=20 --log-file=testdir/ceph-helpers/$name.log --pid-file=testdir/ceph-helpers/$name.pid ' ../qa/workunits/ceph-helpers.sh:505: activate_osd: ceph-disk --statedir=testdir/ceph-helpers --sysconfdir=testdir/ceph-helpers --prepend-to-path= activate --mark-init=none testdir/ceph-helpers/0 *** DEVELOPER MODE: setting PATH, PYTHONPATH and LD_LIBRARY_PATH *** *** DEVELOPER MODE: setting PATH, PYTHONPATH and LD_LIBRARY_PATH *** got monmap epoch 1 2015-07-15 19:58:53.818201 7f3e2b2457c0 -1 journal FileJournal::_open: disabling aio for non-block journal. Use journal_force_aio to force use of aio anyway 2015-07-15 19:58:53.950997 7f3e2b2457c0 -1 journal FileJournal::_open: disabling aio for non-block journal. Use journal_force_aio to force use of aio anyway 2015-07-15 19:58:53.951525 7f3e2b2457c0 -1 filestore(testdir/ceph-helpers/0) could not find -1/23c2fcde/osd_superblock/0 in index: (2) No such file or directory 2015-07-15 19:58:54.044609 7f3e2b2457c0 -1 created object store testdir/ceph-helpers/0 journal testdir/ceph-helpers/0/journal for osd.0 fsid 3f1b7cce-0c19-4c66-990c-9bbd1a39c039 2015-07-15 19:58:54.044669 7f3e2b2457c0 -1 auth: error reading file: testdir/ceph-helpers/0/keyring: can't open testdir/ceph-helpers/0/keyring: (2) No such file or directory 2015-07-15 19:58:54.044940 7f3e2b2457c0 -1 created new key in keyring testdir/ceph-helpers/0/keyring *** DEVELOPER MODE: setting PATH, PYTHONPATH and LD_LIBRARY_PATH *** added key for osd.0 starting osd.0 at :/0 osd_data testdir/ceph-helpers/0 testdir/ceph-helpers/0/journal .../qa/workunits/ceph-helpers.sh:510: activate_osd: cat testdir/ceph-helpers/0/whoami ../qa/workunits/ceph-helpers.sh:510: activate_osd: '[' 0 = 0 ']' ../qa/workunits/ceph-helpers.sh:512: activate_osd: ceph osd crush create-or-move 0 1 root=default host=localhost *** DEVELOPER MODE: setting PATH, PYTHONPATH and LD_LIBRARY_PATH *** create-or-move updating item name 'osd.0' weight 1 at location {host=localhost,root=default} to crush map ../qa/workunits/ceph-helpers.sh:514: activate_osd: wait_for_osd up 0 ../qa/workunits/ceph-helpers.sh:550: wait_for_osd: local state=up ../qa/workunits/ceph-helpers.sh:551: wait_for_osd: local id=0 ../qa/workunits/ceph-helpers.sh:553: wait_for_osd: status=1 ../qa/workunits/ceph-helpers.sh:554: wait_for_osd: (( i=0 )) ../qa/workunits/ceph-helpers.sh:554: wait_for_osd: (( i < 120 )) ../qa/workunits/ceph-helpers.sh:555: wait_for_osd: ceph osd dump ../qa/workunits/ceph-helpers.sh:555: wait_for_osd: grep 'osd.0 up' *** DEVELOPER MODE: setting PATH, PYTHONPATH and LD_LIBRARY_PATH *** osd.0 up in weight 1 up_from 5 up_thru 6 down_at 0 last_clean_interval [0,0) 127.0.0.1:6812/31353 127.0.0.1:6813/31353 127.0.0.1:6814/31353 127.0.0.1:6815/31353 exists,up bcb50905-0273-4031-8ea1-50c66b629bb3 ../qa/workunits/ceph-helpers.sh:558: wait_for_osd: status=0 ../qa/workunits/ceph-helpers.sh:559: wait_for_osd: break ../qa/workunits/ceph-helpers.sh:562: wait_for_osd: return 0 .../qa/workunits/ceph-helpers.sh:526: test_activate_osd: CEPH_ARGS= .../qa/workunits/ceph-helpers.sh:526: test_activate_osd: ceph --format=json daemon testdir/ceph-helpers//ceph-osd.0.asok config get osd_max_backfills *** DEVELOPER MODE: setting PATH, PYTHONPATH and LD_LIBRARY_PATH *** ../qa/workunits/ceph-helpers.sh:526: test_activate_osd: local 'backfills={"osd_max_backfills":"10"}' ../qa/workunits/ceph-helpers.sh:527: test_activate_osd: test '{"osd_max_backfills":"10"}' = '{"osd_max_backfills":"10"}' ../qa/workunits/ceph-helpers.sh:529: test_activate_osd: kill_daemons testdir/ceph-helpers TERM osd .../qa/workunits/ceph-helpers.sh:192: kill_daemons: shopt -q -o xtrace .../qa/workunits/ceph-helpers.sh:192: kill_daemons: echo true ../qa/workunits/ceph-helpers.sh:192: kill_daemons: local trace=true ../qa/workunits/ceph-helpers.sh:193: kill_daemons: true ../qa/workunits/ceph-helpers.sh:193: kill_daemons: shopt -u -o xtrace ../qa/workunits/ceph-helpers.sh:219: kill_daemons: return 0 ../qa/workunits/ceph-helpers.sh:531: test_activate_osd: activate_osd testdir/ceph-helpers 0 --osd-max-backfills 20 ../qa/workunits/ceph-helpers.sh:479: activate_osd: local dir=testdir/ceph-helpers ../qa/workunits/ceph-helpers.sh:480: activate_osd: shift ../qa/workunits/ceph-helpers.sh:481: activate_osd: local id=0 ../qa/workunits/ceph-helpers.sh:482: activate_osd: shift ../qa/workunits/ceph-helpers.sh:483: activate_osd: local osd_data=testdir/ceph-helpers/0 ../qa/workunits/ceph-helpers.sh:485: activate_osd: local ceph_disk_args ../qa/workunits/ceph-helpers.sh:486: activate_osd: ceph_disk_args+=' --statedir=testdir/ceph-helpers' ../qa/workunits/ceph-helpers.sh:487: activate_osd: ceph_disk_args+=' --sysconfdir=testdir/ceph-helpers' ../qa/workunits/ceph-helpers.sh:488: activate_osd: ceph_disk_args+=' --prepend-to-path=' ../qa/workunits/ceph-helpers.sh:490: activate_osd: local 'ceph_args=--fsid=3f1b7cce-0c19-4c66-990c-9bbd1a39c039 --auth-supported=none --mon-host=127.0.0.1:7109 ' ../qa/workunits/ceph-helpers.sh:491: activate_osd: ceph_args+=' --osd-backfill-full-ratio=.99' ../qa/workunits/ceph-helpers.sh:492: activate_osd: ceph_args+=' --osd-failsafe-full-ratio=.99' ../qa/workunits/ceph-helpers.sh:493: activate_osd: ceph_args+=' --osd-journal-size=100' ../qa/workunits/ceph-helpers.sh:494: activate_osd: ceph_args+=' --osd-data=testdir/ceph-helpers/0' ../qa/workunits/ceph-helpers.sh:495: activate_osd: ceph_args+=' --chdir=' ../qa/workunits/ceph-helpers.sh:496: activate_osd: ceph_args+=' --osd-pool-default-erasure-code-directory=.libs' ../qa/workunits/ceph-helpers.sh:497: activate_osd: ceph_args+=' --osd-class-dir=.libs' ../qa/workunits/ceph-helpers.sh:498: activate_osd: ceph_args+=' --run-dir=testdir/ceph-helpers' ../qa/workunits/ceph-helpers.sh:499: activate_osd: ceph_args+=' --debug-osd=20' ../qa/workunits/ceph-helpers.sh:500: activate_osd: ceph_args+=' --log-file=testdir/ceph-helpers/$name.log' ../qa/workunits/ceph-helpers.sh:501: activate_osd: ceph_args+=' --pid-file=testdir/ceph-helpers/$name.pid' ../qa/workunits/ceph-helpers.sh:502: activate_osd: ceph_args+=' ' ../qa/workunits/ceph-helpers.sh:503: activate_osd: ceph_args+='--osd-max-backfills 20' ../qa/workunits/ceph-helpers.sh:504: activate_osd: mkdir -p testdir/ceph-helpers/0 ../qa/workunits/ceph-helpers.sh:505: activate_osd: CEPH_ARGS='--fsid=3f1b7cce-0c19-4c66-990c-9bbd1a39c039 --auth-supported=none --mon-host=127.0.0.1:7109 --osd-backfill-full-ratio=.99 --osd-failsafe-full-ratio=.99 --osd-journal-size=100 --osd-data=testdir/ceph-helpers/0 --chdir= --osd-pool-default-erasure-code-directory=.libs --osd-class-dir=.libs --run-dir=testdir/ceph-helpers --debug-osd=20 --log-file=testdir/ceph-helpers/$name.log --pid-file=testdir/ceph-helpers/$name.pid --osd-max-backfills 20 ' ../qa/workunits/ceph-helpers.sh:505: activate_osd: ceph-disk --statedir=testdir/ceph-helpers --sysconfdir=testdir/ceph-helpers --prepend-to-path= activate --mark-init=none testdir/ceph-helpers/0 starting osd.0 at :/0 osd_data testdir/ceph-helpers/0 testdir/ceph-helpers/0/journal .../qa/workunits/ceph-helpers.sh:510: activate_osd: cat testdir/ceph-helpers/0/whoami ../qa/workunits/ceph-helpers.sh:510: activate_osd: '[' 0 = 0 ']' ../qa/workunits/ceph-helpers.sh:512: activate_osd: ceph osd crush create-or-move 0 1 root=default host=localhost *** DEVELOPER MODE: setting PATH, PYTHONPATH and LD_LIBRARY_PATH *** create-or-move updated item name 'osd.0' weight 1 at location {host=localhost,root=default} to crush map ../qa/workunits/ceph-helpers.sh:514: activate_osd: wait_for_osd up 0 ../qa/workunits/ceph-helpers.sh:550: wait_for_osd: local state=up ../qa/workunits/ceph-helpers.sh:551: wait_for_osd: local id=0 ../qa/workunits/ceph-helpers.sh:553: wait_for_osd: status=1 ../qa/workunits/ceph-helpers.sh:554: wait_for_osd: (( i=0 )) ../qa/workunits/ceph-helpers.sh:554: wait_for_osd: (( i < 120 )) ../qa/workunits/ceph-helpers.sh:555: wait_for_osd: ceph osd dump ../qa/workunits/ceph-helpers.sh:555: wait_for_osd: grep 'osd.0 up' *** DEVELOPER MODE: setting PATH, PYTHONPATH and LD_LIBRARY_PATH *** osd.0 up in weight 1 up_from 9 up_thru 9 down_at 8 last_clean_interval [5,7) 127.0.0.1:6852/4292 127.0.0.1:6853/4292 127.0.0.1:6854/4292 127.0.0.1:6855/4292 exists,up bcb50905-0273-4031-8ea1-50c66b629bb3 ../qa/workunits/ceph-helpers.sh:558: wait_for_osd: status=0 ../qa/workunits/ceph-helpers.sh:559: wait_for_osd: break ../qa/workunits/ceph-helpers.sh:562: wait_for_osd: return 0 .../qa/workunits/ceph-helpers.sh:533: test_activate_osd: CEPH_ARGS= .../qa/workunits/ceph-helpers.sh:533: test_activate_osd: ceph --format=json daemon testdir/ceph-helpers//ceph-osd.0.asok config get osd_max_backfills *** DEVELOPER MODE: setting PATH, PYTHONPATH and LD_LIBRARY_PATH *** ../qa/workunits/ceph-helpers.sh:533: test_activate_osd: local 'backfills={"osd_max_backfills":"20"}' ../qa/workunits/ceph-helpers.sh:534: test_activate_osd: test '{"osd_max_backfills":"20"}' = '{"osd_max_backfills":"20"}' ../qa/workunits/ceph-helpers.sh:536: test_activate_osd: teardown testdir/ceph-helpers ../qa/workunits/ceph-helpers.sh:117: teardown: local dir=testdir/ceph-helpers ../qa/workunits/ceph-helpers.sh:118: teardown: kill_daemons testdir/ceph-helpers .../qa/workunits/ceph-helpers.sh:192: kill_daemons: shopt -q -o xtrace .../qa/workunits/ceph-helpers.sh:192: kill_daemons: echo true ../qa/workunits/ceph-helpers.sh:192: kill_daemons: local trace=true ../qa/workunits/ceph-helpers.sh:193: kill_daemons: true ../qa/workunits/ceph-helpers.sh:193: kill_daemons: shopt -u -o xtrace ../qa/workunits/ceph-helpers.sh:219: kill_daemons: return 0 .../qa/workunits/ceph-helpers.sh:119: teardown: stat -f -c %T . ../qa/workunits/ceph-helpers.sh:119: teardown: '[' ext2/ext3 == btrfs ']' ../qa/workunits/ceph-helpers.sh:122: teardown: rm -fr testdir/ceph-helpers ../qa/workunits/ceph-helpers.sh:1198: run_tests: for func in '$funcs' ../qa/workunits/ceph-helpers.sh:1199: run_tests: test_erasure_code_plugin_exists testdir/ceph-helpers ../qa/workunits/ceph-helpers.sh:1129: test_erasure_code_plugin_exists: local dir=testdir/ceph-helpers ../qa/workunits/ceph-helpers.sh:1131: test_erasure_code_plugin_exists: setup testdir/ceph-helpers ../qa/workunits/ceph-helpers.sh:92: setup: local dir=testdir/ceph-helpers ../qa/workunits/ceph-helpers.sh:93: setup: teardown testdir/ceph-helpers ../qa/workunits/ceph-helpers.sh:117: teardown: local dir=testdir/ceph-helpers ../qa/workunits/ceph-helpers.sh:118: teardown: kill_daemons testdir/ceph-helpers .../qa/workunits/ceph-helpers.sh:192: kill_daemons: shopt -q -o xtrace .../qa/workunits/ceph-helpers.sh:192: kill_daemons: echo true ../qa/workunits/ceph-helpers.sh:192: kill_daemons: local trace=true ../qa/workunits/ceph-helpers.sh:193: kill_daemons: true ../qa/workunits/ceph-helpers.sh:193: kill_daemons: shopt -u -o xtrace ../qa/workunits/ceph-helpers.sh:219: kill_daemons: return 0 .../qa/workunits/ceph-helpers.sh:119: teardown: stat -f -c %T . ../qa/workunits/ceph-helpers.sh:119: teardown: '[' ext2/ext3 == btrfs ']' ../qa/workunits/ceph-helpers.sh:122: teardown: rm -fr testdir/ceph-helpers ../qa/workunits/ceph-helpers.sh:94: setup: mkdir -p testdir/ceph-helpers ../qa/workunits/ceph-helpers.sh:1132: test_erasure_code_plugin_exists: run_mon testdir/ceph-helpers a ../qa/workunits/ceph-helpers.sh:282: run_mon: local dir=testdir/ceph-helpers ../qa/workunits/ceph-helpers.sh:283: run_mon: shift ../qa/workunits/ceph-helpers.sh:284: run_mon: local id=a ../qa/workunits/ceph-helpers.sh:285: run_mon: shift ../qa/workunits/ceph-helpers.sh:286: run_mon: local data=testdir/ceph-helpers/a ../qa/workunits/ceph-helpers.sh:289: run_mon: ceph-mon --id a --mkfs --mon-data=testdir/ceph-helpers/a --run-dir=testdir/ceph-helpers ceph-mon: mon.noname-a 127.0.0.1:7109/0 is local, renaming to mon.a ceph-mon: set fsid to 3f1b7cce-0c19-4c66-990c-9bbd1a39c039 ceph-mon: created monfs at testdir/ceph-helpers/a for mon.a ../qa/workunits/ceph-helpers.sh:296: run_mon: ceph-mon --id a --mon-osd-full-ratio=.99 --mon-data-avail-crit=1 --paxos-propose-interval=0.1 --osd-crush-chooseleaf-type=0 --osd-pool-default-erasure-code-directory=.libs --debug-mon 20 --debug-ms 20 --debug-paxos 20 --chdir= --mon-data=testdir/ceph-helpers/a '--log-file=testdir/ceph-helpers/$name.log' '--admin-socket=testdir/ceph-helpers/$cluster-$name.asok' --mon-cluster-log-file=testdir/ceph-helpers/log --run-dir=testdir/ceph-helpers '--pid-file=testdir/ceph-helpers/$name.pid' ../qa/workunits/ceph-helpers.sh:314: run_mon: cat .../qa/workunits/ceph-helpers.sh:314: run_mon: get_config mon a fsid .../qa/workunits/ceph-helpers.sh:653: get_config: local daemon=mon .../qa/workunits/ceph-helpers.sh:654: get_config: local id=a .../qa/workunits/ceph-helpers.sh:655: get_config: local config=fsid .../qa/workunits/ceph-helpers.sh:657: get_config: CEPH_ARGS= .../qa/workunits/ceph-helpers.sh:657: get_config: ceph --format xml daemon testdir/ceph-helpers/ceph-mon.a.asok config get fsid .../qa/workunits/ceph-helpers.sh:660: get_config: xmlstarlet sel -t -m //fsid -v . -n .../qa/workunits/ceph-helpers.sh:314: run_mon: get_config mon a mon_host .../qa/workunits/ceph-helpers.sh:653: get_config: local daemon=mon .../qa/workunits/ceph-helpers.sh:654: get_config: local id=a .../qa/workunits/ceph-helpers.sh:655: get_config: local config=mon_host .../qa/workunits/ceph-helpers.sh:657: get_config: CEPH_ARGS= .../qa/workunits/ceph-helpers.sh:657: get_config: ceph --format xml daemon testdir/ceph-helpers/ceph-mon.a.asok config get mon_host .../qa/workunits/ceph-helpers.sh:660: get_config: xmlstarlet sel -t -m //mon_host -v . -n .../qa/workunits/ceph-helpers.sh:319: run_mon: get_config mon a mon_initial_members .../qa/workunits/ceph-helpers.sh:653: get_config: local daemon=mon .../qa/workunits/ceph-helpers.sh:654: get_config: local id=a .../qa/workunits/ceph-helpers.sh:655: get_config: local config=mon_initial_members .../qa/workunits/ceph-helpers.sh:657: get_config: CEPH_ARGS= .../qa/workunits/ceph-helpers.sh:657: get_config: ceph --format xml daemon testdir/ceph-helpers/ceph-mon.a.asok config get mon_initial_members .../qa/workunits/ceph-helpers.sh:660: get_config: xmlstarlet sel -t -m //mon_initial_members -v . -n ../qa/workunits/ceph-helpers.sh:319: run_mon: test -z '' ../qa/workunits/ceph-helpers.sh:320: run_mon: ceph osd pool delete rbd rbd --yes-i-really-really-mean-it *** DEVELOPER MODE: setting PATH, PYTHONPATH and LD_LIBRARY_PATH *** pool 'rbd' removed ../qa/workunits/ceph-helpers.sh:321: run_mon: ceph osd pool create rbd 4 *** DEVELOPER MODE: setting PATH, PYTHONPATH and LD_LIBRARY_PATH *** pool 'rbd' created ../qa/workunits/ceph-helpers.sh:1133: test_erasure_code_plugin_exists: erasure_code_plugin_exists jerasure ../qa/workunits/ceph-helpers.sh:1115: erasure_code_plugin_exists: local plugin=jerasure ../qa/workunits/ceph-helpers.sh:1117: erasure_code_plugin_exists: local status ../qa/workunits/ceph-helpers.sh:1118: erasure_code_plugin_exists: ceph osd erasure-code-profile set TESTPROFILE plugin=jerasure ../qa/workunits/ceph-helpers.sh:1119: erasure_code_plugin_exists: grep 'jerasure.*No such file' ../qa/workunits/ceph-helpers.sh:1122: erasure_code_plugin_exists: status=0 ../qa/workunits/ceph-helpers.sh:1123: erasure_code_plugin_exists: ceph osd erasure-code-profile rm TESTPROFILE *** DEVELOPER MODE: setting PATH, PYTHONPATH and LD_LIBRARY_PATH *** ../qa/workunits/ceph-helpers.sh:1125: erasure_code_plugin_exists: return 0 ../qa/workunits/ceph-helpers.sh:1134: test_erasure_code_plugin_exists: erasure_code_plugin_exists FAKE ../qa/workunits/ceph-helpers.sh:1115: erasure_code_plugin_exists: local plugin=FAKE ../qa/workunits/ceph-helpers.sh:1117: erasure_code_plugin_exists: local status ../qa/workunits/ceph-helpers.sh:1118: erasure_code_plugin_exists: ceph osd erasure-code-profile set TESTPROFILE plugin=FAKE ../qa/workunits/ceph-helpers.sh:1119: erasure_code_plugin_exists: grep 'FAKE.*No such file' Error EIO: load dlopen(.libs/libec_FAKE.so): .libs/libec_FAKE.so: cannot open shared object file: No such file or directory ../qa/workunits/ceph-helpers.sh:1120: erasure_code_plugin_exists: status=1 ../qa/workunits/ceph-helpers.sh:1125: erasure_code_plugin_exists: return 1 ../qa/workunits/ceph-helpers.sh:1135: test_erasure_code_plugin_exists: teardown testdir/ceph-helpers ../qa/workunits/ceph-helpers.sh:117: teardown: local dir=testdir/ceph-helpers ../qa/workunits/ceph-helpers.sh:118: teardown: kill_daemons testdir/ceph-helpers .../qa/workunits/ceph-helpers.sh:192: kill_daemons: shopt -q -o xtrace .../qa/workunits/ceph-helpers.sh:192: kill_daemons: echo true ../qa/workunits/ceph-helpers.sh:192: kill_daemons: local trace=true ../qa/workunits/ceph-helpers.sh:193: kill_daemons: true ../qa/workunits/ceph-helpers.sh:193: kill_daemons: shopt -u -o xtrace ../qa/workunits/ceph-helpers.sh:219: kill_daemons: return 0 .../qa/workunits/ceph-helpers.sh:119: teardown: stat -f -c %T . ../qa/workunits/ceph-helpers.sh:119: teardown: '[' ext2/ext3 == btrfs ']' ../qa/workunits/ceph-helpers.sh:122: teardown: rm -fr testdir/ceph-helpers ../qa/workunits/ceph-helpers.sh:1198: run_tests: for func in '$funcs' ../qa/workunits/ceph-helpers.sh:1199: run_tests: test_expect_failure testdir/ceph-helpers ../qa/workunits/ceph-helpers.sh:1092: test_expect_failure: local dir=testdir/ceph-helpers ../qa/workunits/ceph-helpers.sh:1094: test_expect_failure: setup testdir/ceph-helpers ../qa/workunits/ceph-helpers.sh:92: setup: local dir=testdir/ceph-helpers ../qa/workunits/ceph-helpers.sh:93: setup: teardown testdir/ceph-helpers ../qa/workunits/ceph-helpers.sh:117: teardown: local dir=testdir/ceph-helpers ../qa/workunits/ceph-helpers.sh:118: teardown: kill_daemons testdir/ceph-helpers .../qa/workunits/ceph-helpers.sh:192: kill_daemons: shopt -q -o xtrace .../qa/workunits/ceph-helpers.sh:192: kill_daemons: echo true ../qa/workunits/ceph-helpers.sh:192: kill_daemons: local trace=true ../qa/workunits/ceph-helpers.sh:193: kill_daemons: true ../qa/workunits/ceph-helpers.sh:193: kill_daemons: shopt -u -o xtrace ../qa/workunits/ceph-helpers.sh:219: kill_daemons: return 0 .../qa/workunits/ceph-helpers.sh:119: teardown: stat -f -c %T . ../qa/workunits/ceph-helpers.sh:119: teardown: '[' ext2/ext3 == btrfs ']' ../qa/workunits/ceph-helpers.sh:122: teardown: rm -fr testdir/ceph-helpers ../qa/workunits/ceph-helpers.sh:94: setup: mkdir -p testdir/ceph-helpers ../qa/workunits/ceph-helpers.sh:1095: test_expect_failure: expect_failure testdir/ceph-helpers FAIL bash -c 'echo FAIL ; exit 1' ../qa/workunits/ceph-helpers.sh:1071: expect_failure: local dir=testdir/ceph-helpers ../qa/workunits/ceph-helpers.sh:1072: expect_failure: shift ../qa/workunits/ceph-helpers.sh:1073: expect_failure: local expected=FAIL ../qa/workunits/ceph-helpers.sh:1074: expect_failure: shift ../qa/workunits/ceph-helpers.sh:1075: expect_failure: local success ../qa/workunits/ceph-helpers.sh:1077: expect_failure: bash -c 'echo FAIL ; exit 1' ../qa/workunits/ceph-helpers.sh:1080: expect_failure: success=false ../qa/workunits/ceph-helpers.sh:1083: expect_failure: false ../qa/workunits/ceph-helpers.sh:1083: expect_failure: grep --quiet FAIL testdir/ceph-helpers/out ../qa/workunits/ceph-helpers.sh:1087: expect_failure: return 0 ../qa/workunits/ceph-helpers.sh:1097: test_expect_failure: expect_failure testdir/ceph-helpers FAIL bash -c 'echo FAIL ; exit 0' ../qa/workunits/ceph-helpers.sh:1071: expect_failure: local dir=testdir/ceph-helpers ../qa/workunits/ceph-helpers.sh:1072: expect_failure: shift ../qa/workunits/ceph-helpers.sh:1073: expect_failure: local expected=FAIL ../qa/workunits/ceph-helpers.sh:1074: expect_failure: shift ../qa/workunits/ceph-helpers.sh:1075: expect_failure: local success ../qa/workunits/ceph-helpers.sh:1077: expect_failure: bash -c 'echo FAIL ; exit 0' ../qa/workunits/ceph-helpers.sh:1078: expect_failure: success=true ../qa/workunits/ceph-helpers.sh:1083: expect_failure: true ../qa/workunits/ceph-helpers.sh:1084: expect_failure: cat testdir/ceph-helpers/out FAIL ../qa/workunits/ceph-helpers.sh:1085: expect_failure: return 1 ../qa/workunits/ceph-helpers.sh:1098: test_expect_failure: grep --quiet FAIL testdir/ceph-helpers/out ../qa/workunits/ceph-helpers.sh:1100: test_expect_failure: expect_failure testdir/ceph-helpers FAIL bash -c 'echo UNEXPECTED ; exit 1' ../qa/workunits/ceph-helpers.sh:1071: expect_failure: local dir=testdir/ceph-helpers ../qa/workunits/ceph-helpers.sh:1072: expect_failure: shift ../qa/workunits/ceph-helpers.sh:1073: expect_failure: local expected=FAIL ../qa/workunits/ceph-helpers.sh:1074: expect_failure: shift ../qa/workunits/ceph-helpers.sh:1075: expect_failure: local success ../qa/workunits/ceph-helpers.sh:1077: expect_failure: bash -c 'echo UNEXPECTED ; exit 1' ../qa/workunits/ceph-helpers.sh:1080: expect_failure: success=false ../qa/workunits/ceph-helpers.sh:1083: expect_failure: false ../qa/workunits/ceph-helpers.sh:1083: expect_failure: grep --quiet FAIL testdir/ceph-helpers/out ../qa/workunits/ceph-helpers.sh:1084: expect_failure: cat testdir/ceph-helpers/out UNEXPECTED ../qa/workunits/ceph-helpers.sh:1085: expect_failure: return 1 ../qa/workunits/ceph-helpers.sh:1101: test_expect_failure: grep --quiet FAIL testdir/ceph-helpers/out ../qa/workunits/ceph-helpers.sh:1102: test_expect_failure: teardown testdir/ceph-helpers ../qa/workunits/ceph-helpers.sh:117: teardown: local dir=testdir/ceph-helpers ../qa/workunits/ceph-helpers.sh:118: teardown: kill_daemons testdir/ceph-helpers .../qa/workunits/ceph-helpers.sh:192: kill_daemons: shopt -q -o xtrace .../qa/workunits/ceph-helpers.sh:192: kill_daemons: echo true ../qa/workunits/ceph-helpers.sh:192: kill_daemons: local trace=true ../qa/workunits/ceph-helpers.sh:193: kill_daemons: true ../qa/workunits/ceph-helpers.sh:193: kill_daemons: shopt -u -o xtrace ../qa/workunits/ceph-helpers.sh:219: kill_daemons: return 0 .../qa/workunits/ceph-helpers.sh:119: teardown: stat -f -c %T . ../qa/workunits/ceph-helpers.sh:119: teardown: '[' ext2/ext3 == btrfs ']' ../qa/workunits/ceph-helpers.sh:122: teardown: rm -fr testdir/ceph-helpers ../qa/workunits/ceph-helpers.sh:1198: run_tests: for func in '$funcs' ../qa/workunits/ceph-helpers.sh:1199: run_tests: test_get_config testdir/ceph-helpers ../qa/workunits/ceph-helpers.sh:664: test_get_config: local dir=testdir/ceph-helpers ../qa/workunits/ceph-helpers.sh:667: test_get_config: setup testdir/ceph-helpers ../qa/workunits/ceph-helpers.sh:92: setup: local dir=testdir/ceph-helpers ../qa/workunits/ceph-helpers.sh:93: setup: teardown testdir/ceph-helpers ../qa/workunits/ceph-helpers.sh:117: teardown: local dir=testdir/ceph-helpers ../qa/workunits/ceph-helpers.sh:118: teardown: kill_daemons testdir/ceph-helpers .../qa/workunits/ceph-helpers.sh:192: kill_daemons: shopt -q -o xtrace .../qa/workunits/ceph-helpers.sh:192: kill_daemons: echo true ../qa/workunits/ceph-helpers.sh:192: kill_daemons: local trace=true ../qa/workunits/ceph-helpers.sh:193: kill_daemons: true ../qa/workunits/ceph-helpers.sh:193: kill_daemons: shopt -u -o xtrace ../qa/workunits/ceph-helpers.sh:219: kill_daemons: return 0 .../qa/workunits/ceph-helpers.sh:119: teardown: stat -f -c %T . ../qa/workunits/ceph-helpers.sh:119: teardown: '[' ext2/ext3 == btrfs ']' ../qa/workunits/ceph-helpers.sh:122: teardown: rm -fr testdir/ceph-helpers ../qa/workunits/ceph-helpers.sh:94: setup: mkdir -p testdir/ceph-helpers ../qa/workunits/ceph-helpers.sh:668: test_get_config: run_mon testdir/ceph-helpers a --osd_pool_default_size=1 ../qa/workunits/ceph-helpers.sh:282: run_mon: local dir=testdir/ceph-helpers ../qa/workunits/ceph-helpers.sh:283: run_mon: shift ../qa/workunits/ceph-helpers.sh:284: run_mon: local id=a ../qa/workunits/ceph-helpers.sh:285: run_mon: shift ../qa/workunits/ceph-helpers.sh:286: run_mon: local data=testdir/ceph-helpers/a ../qa/workunits/ceph-helpers.sh:289: run_mon: ceph-mon --id a --mkfs --mon-data=testdir/ceph-helpers/a --run-dir=testdir/ceph-helpers --osd_pool_default_size=1 ceph-mon: mon.noname-a 127.0.0.1:7109/0 is local, renaming to mon.a ceph-mon: set fsid to 3f1b7cce-0c19-4c66-990c-9bbd1a39c039 ceph-mon: created monfs at testdir/ceph-helpers/a for mon.a ../qa/workunits/ceph-helpers.sh:296: run_mon: ceph-mon --id a --mon-osd-full-ratio=.99 --mon-data-avail-crit=1 --paxos-propose-interval=0.1 --osd-crush-chooseleaf-type=0 --osd-pool-default-erasure-code-directory=.libs --debug-mon 20 --debug-ms 20 --debug-paxos 20 --chdir= --mon-data=testdir/ceph-helpers/a '--log-file=testdir/ceph-helpers/$name.log' '--admin-socket=testdir/ceph-helpers/$cluster-$name.asok' --mon-cluster-log-file=testdir/ceph-helpers/log --run-dir=testdir/ceph-helpers '--pid-file=testdir/ceph-helpers/$name.pid' --osd_pool_default_size=1 ../qa/workunits/ceph-helpers.sh:314: run_mon: cat .../qa/workunits/ceph-helpers.sh:314: run_mon: get_config mon a fsid .../qa/workunits/ceph-helpers.sh:653: get_config: local daemon=mon .../qa/workunits/ceph-helpers.sh:654: get_config: local id=a .../qa/workunits/ceph-helpers.sh:655: get_config: local config=fsid .../qa/workunits/ceph-helpers.sh:657: get_config: CEPH_ARGS= .../qa/workunits/ceph-helpers.sh:657: get_config: ceph --format xml daemon testdir/ceph-helpers/ceph-mon.a.asok config get fsid .../qa/workunits/ceph-helpers.sh:660: get_config: xmlstarlet sel -t -m //fsid -v . -n .../qa/workunits/ceph-helpers.sh:314: run_mon: get_config mon a mon_host .../qa/workunits/ceph-helpers.sh:653: get_config: local daemon=mon .../qa/workunits/ceph-helpers.sh:654: get_config: local id=a .../qa/workunits/ceph-helpers.sh:655: get_config: local config=mon_host .../qa/workunits/ceph-helpers.sh:657: get_config: CEPH_ARGS= .../qa/workunits/ceph-helpers.sh:657: get_config: ceph --format xml daemon testdir/ceph-helpers/ceph-mon.a.asok config get mon_host .../qa/workunits/ceph-helpers.sh:660: get_config: xmlstarlet sel -t -m //mon_host -v . -n .../qa/workunits/ceph-helpers.sh:319: run_mon: get_config mon a mon_initial_members .../qa/workunits/ceph-helpers.sh:653: get_config: local daemon=mon .../qa/workunits/ceph-helpers.sh:654: get_config: local id=a .../qa/workunits/ceph-helpers.sh:655: get_config: local config=mon_initial_members .../qa/workunits/ceph-helpers.sh:657: get_config: CEPH_ARGS= .../qa/workunits/ceph-helpers.sh:657: get_config: ceph --format xml daemon testdir/ceph-helpers/ceph-mon.a.asok config get mon_initial_members .../qa/workunits/ceph-helpers.sh:660: get_config: xmlstarlet sel -t -m //mon_initial_members -v . -n ../qa/workunits/ceph-helpers.sh:319: run_mon: test -z '' ../qa/workunits/ceph-helpers.sh:320: run_mon: ceph osd pool delete rbd rbd --yes-i-really-really-mean-it *** DEVELOPER MODE: setting PATH, PYTHONPATH and LD_LIBRARY_PATH *** pool 'rbd' removed ../qa/workunits/ceph-helpers.sh:321: run_mon: ceph osd pool create rbd 4 *** DEVELOPER MODE: setting PATH, PYTHONPATH and LD_LIBRARY_PATH *** pool 'rbd' created .../qa/workunits/ceph-helpers.sh:669: test_get_config: get_config mon a osd_pool_default_size .../qa/workunits/ceph-helpers.sh:653: get_config: local daemon=mon .../qa/workunits/ceph-helpers.sh:654: get_config: local id=a .../qa/workunits/ceph-helpers.sh:655: get_config: local config=osd_pool_default_size .../qa/workunits/ceph-helpers.sh:660: get_config: xmlstarlet sel -t -m //osd_pool_default_size -v . -n .../qa/workunits/ceph-helpers.sh:657: get_config: CEPH_ARGS= .../qa/workunits/ceph-helpers.sh:657: get_config: ceph --format xml daemon testdir/ceph-helpers/ceph-mon.a.asok config get osd_pool_default_size ../qa/workunits/ceph-helpers.sh:669: test_get_config: test 1 = 1 ../qa/workunits/ceph-helpers.sh:670: test_get_config: run_osd testdir/ceph-helpers 0 --osd_max_scrubs=3 ../qa/workunits/ceph-helpers.sh:397: run_osd: local dir=testdir/ceph-helpers ../qa/workunits/ceph-helpers.sh:398: run_osd: shift ../qa/workunits/ceph-helpers.sh:399: run_osd: local id=0 ../qa/workunits/ceph-helpers.sh:400: run_osd: shift ../qa/workunits/ceph-helpers.sh:401: run_osd: local osd_data=testdir/ceph-helpers/0 ../qa/workunits/ceph-helpers.sh:403: run_osd: local ceph_disk_args ../qa/workunits/ceph-helpers.sh:404: run_osd: ceph_disk_args+=' --statedir=testdir/ceph-helpers' ../qa/workunits/ceph-helpers.sh:405: run_osd: ceph_disk_args+=' --sysconfdir=testdir/ceph-helpers' ../qa/workunits/ceph-helpers.sh:406: run_osd: ceph_disk_args+=' --prepend-to-path=' ../qa/workunits/ceph-helpers.sh:408: run_osd: mkdir -p testdir/ceph-helpers/0 ../qa/workunits/ceph-helpers.sh:409: run_osd: ceph-disk --statedir=testdir/ceph-helpers --sysconfdir=testdir/ceph-helpers --prepend-to-path= prepare testdir/ceph-helpers/0 2015-07-15 19:59:08.338828 7fadc81c37c0 -1 warning: unable to create /var/run/ceph: (13) Permission denied 2015-07-15 19:59:08.352391 7f6d0f9687c0 -1 warning: unable to create /var/run/ceph: (13) Permission denied 2015-07-15 19:59:08.366205 7fd04b0b27c0 -1 warning: unable to create /var/run/ceph: (13) Permission denied ../qa/workunits/ceph-helpers.sh:412: run_osd: activate_osd testdir/ceph-helpers 0 --osd_max_scrubs=3 ../qa/workunits/ceph-helpers.sh:479: activate_osd: local dir=testdir/ceph-helpers ../qa/workunits/ceph-helpers.sh:480: activate_osd: shift ../qa/workunits/ceph-helpers.sh:481: activate_osd: local id=0 ../qa/workunits/ceph-helpers.sh:482: activate_osd: shift ../qa/workunits/ceph-helpers.sh:483: activate_osd: local osd_data=testdir/ceph-helpers/0 ../qa/workunits/ceph-helpers.sh:485: activate_osd: local ceph_disk_args ../qa/workunits/ceph-helpers.sh:486: activate_osd: ceph_disk_args+=' --statedir=testdir/ceph-helpers' ../qa/workunits/ceph-helpers.sh:487: activate_osd: ceph_disk_args+=' --sysconfdir=testdir/ceph-helpers' ../qa/workunits/ceph-helpers.sh:488: activate_osd: ceph_disk_args+=' --prepend-to-path=' ../qa/workunits/ceph-helpers.sh:490: activate_osd: local 'ceph_args=--fsid=3f1b7cce-0c19-4c66-990c-9bbd1a39c039 --auth-supported=none --mon-host=127.0.0.1:7109 ' ../qa/workunits/ceph-helpers.sh:491: activate_osd: ceph_args+=' --osd-backfill-full-ratio=.99' ../qa/workunits/ceph-helpers.sh:492: activate_osd: ceph_args+=' --osd-failsafe-full-ratio=.99' ../qa/workunits/ceph-helpers.sh:493: activate_osd: ceph_args+=' --osd-journal-size=100' ../qa/workunits/ceph-helpers.sh:494: activate_osd: ceph_args+=' --osd-data=testdir/ceph-helpers/0' ../qa/workunits/ceph-helpers.sh:495: activate_osd: ceph_args+=' --chdir=' ../qa/workunits/ceph-helpers.sh:496: activate_osd: ceph_args+=' --osd-pool-default-erasure-code-directory=.libs' ../qa/workunits/ceph-helpers.sh:497: activate_osd: ceph_args+=' --osd-class-dir=.libs' ../qa/workunits/ceph-helpers.sh:498: activate_osd: ceph_args+=' --run-dir=testdir/ceph-helpers' ../qa/workunits/ceph-helpers.sh:499: activate_osd: ceph_args+=' --debug-osd=20' ../qa/workunits/ceph-helpers.sh:500: activate_osd: ceph_args+=' --log-file=testdir/ceph-helpers/$name.log' ../qa/workunits/ceph-helpers.sh:501: activate_osd: ceph_args+=' --pid-file=testdir/ceph-helpers/$name.pid' ../qa/workunits/ceph-helpers.sh:502: activate_osd: ceph_args+=' ' ../qa/workunits/ceph-helpers.sh:503: activate_osd: ceph_args+=--osd_max_scrubs=3 ../qa/workunits/ceph-helpers.sh:504: activate_osd: mkdir -p testdir/ceph-helpers/0 ../qa/workunits/ceph-helpers.sh:505: activate_osd: CEPH_ARGS='--fsid=3f1b7cce-0c19-4c66-990c-9bbd1a39c039 --auth-supported=none --mon-host=127.0.0.1:7109 --osd-backfill-full-ratio=.99 --osd-failsafe-full-ratio=.99 --osd-journal-size=100 --osd-data=testdir/ceph-helpers/0 --chdir= --osd-pool-default-erasure-code-directory=.libs --osd-class-dir=.libs --run-dir=testdir/ceph-helpers --debug-osd=20 --log-file=testdir/ceph-helpers/$name.log --pid-file=testdir/ceph-helpers/$name.pid --osd_max_scrubs=3 ' ../qa/workunits/ceph-helpers.sh:505: activate_osd: ceph-disk --statedir=testdir/ceph-helpers --sysconfdir=testdir/ceph-helpers --prepend-to-path= activate --mark-init=none testdir/ceph-helpers/0 *** DEVELOPER MODE: setting PATH, PYTHONPATH and LD_LIBRARY_PATH *** *** DEVELOPER MODE: setting PATH, PYTHONPATH and LD_LIBRARY_PATH *** got monmap epoch 1 2015-07-15 19:59:10.796843 7f9b078ae7c0 -1 journal FileJournal::_open: disabling aio for non-block journal. Use journal_force_aio to force use of aio anyway 2015-07-15 19:59:10.853431 7f9b078ae7c0 -1 journal FileJournal::_open: disabling aio for non-block journal. Use journal_force_aio to force use of aio anyway 2015-07-15 19:59:10.853919 7f9b078ae7c0 -1 filestore(testdir/ceph-helpers/0) could not find -1/23c2fcde/osd_superblock/0 in index: (2) No such file or directory 2015-07-15 19:59:10.903248 7f9b078ae7c0 -1 created object store testdir/ceph-helpers/0 journal testdir/ceph-helpers/0/journal for osd.0 fsid 3f1b7cce-0c19-4c66-990c-9bbd1a39c039 2015-07-15 19:59:10.903367 7f9b078ae7c0 -1 auth: error reading file: testdir/ceph-helpers/0/keyring: can't open testdir/ceph-helpers/0/keyring: (2) No such file or directory 2015-07-15 19:59:10.903553 7f9b078ae7c0 -1 created new key in keyring testdir/ceph-helpers/0/keyring *** DEVELOPER MODE: setting PATH, PYTHONPATH and LD_LIBRARY_PATH *** added key for osd.0 starting osd.0 at :/0 osd_data testdir/ceph-helpers/0 testdir/ceph-helpers/0/journal .../qa/workunits/ceph-helpers.sh:510: activate_osd: cat testdir/ceph-helpers/0/whoami ../qa/workunits/ceph-helpers.sh:510: activate_osd: '[' 0 = 0 ']' ../qa/workunits/ceph-helpers.sh:512: activate_osd: ceph osd crush create-or-move 0 1 root=default host=localhost *** DEVELOPER MODE: setting PATH, PYTHONPATH and LD_LIBRARY_PATH *** create-or-move updating item name 'osd.0' weight 1 at location {host=localhost,root=default} to crush map ../qa/workunits/ceph-helpers.sh:514: activate_osd: wait_for_osd up 0 ../qa/workunits/ceph-helpers.sh:550: wait_for_osd: local state=up ../qa/workunits/ceph-helpers.sh:551: wait_for_osd: local id=0 ../qa/workunits/ceph-helpers.sh:553: wait_for_osd: status=1 ../qa/workunits/ceph-helpers.sh:554: wait_for_osd: (( i=0 )) ../qa/workunits/ceph-helpers.sh:554: wait_for_osd: (( i < 120 )) ../qa/workunits/ceph-helpers.sh:555: wait_for_osd: ceph osd dump ../qa/workunits/ceph-helpers.sh:555: wait_for_osd: grep 'osd.0 up' *** DEVELOPER MODE: setting PATH, PYTHONPATH and LD_LIBRARY_PATH *** osd.0 up in weight 1 up_from 5 up_thru 6 down_at 0 last_clean_interval [0,0) 127.0.0.1:6878/18192 127.0.0.1:6879/18192 127.0.0.1:6880/18192 127.0.0.1:6881/18192 exists,up dfe1c114-a792-49d9-9c8f-82bbc71a263e ../qa/workunits/ceph-helpers.sh:558: wait_for_osd: status=0 ../qa/workunits/ceph-helpers.sh:559: wait_for_osd: break ../qa/workunits/ceph-helpers.sh:562: wait_for_osd: return 0 .../qa/workunits/ceph-helpers.sh:671: test_get_config: get_config osd 0 osd_max_scrubs .../qa/workunits/ceph-helpers.sh:653: get_config: local daemon=osd .../qa/workunits/ceph-helpers.sh:654: get_config: local id=0 .../qa/workunits/ceph-helpers.sh:655: get_config: local config=osd_max_scrubs .../qa/workunits/ceph-helpers.sh:657: get_config: CEPH_ARGS= .../qa/workunits/ceph-helpers.sh:657: get_config: ceph --format xml daemon testdir/ceph-helpers/ceph-osd.0.asok config get osd_max_scrubs .../qa/workunits/ceph-helpers.sh:660: get_config: xmlstarlet sel -t -m //osd_max_scrubs -v . -n ../qa/workunits/ceph-helpers.sh:671: test_get_config: test 3 = 3 ../qa/workunits/ceph-helpers.sh:672: test_get_config: teardown testdir/ceph-helpers ../qa/workunits/ceph-helpers.sh:117: teardown: local dir=testdir/ceph-helpers ../qa/workunits/ceph-helpers.sh:118: teardown: kill_daemons testdir/ceph-helpers .../qa/workunits/ceph-helpers.sh:192: kill_daemons: shopt -q -o xtrace .../qa/workunits/ceph-helpers.sh:192: kill_daemons: echo true ../qa/workunits/ceph-helpers.sh:192: kill_daemons: local trace=true ../qa/workunits/ceph-helpers.sh:193: kill_daemons: true ../qa/workunits/ceph-helpers.sh:193: kill_daemons: shopt -u -o xtrace ../qa/workunits/ceph-helpers.sh:219: kill_daemons: return 0 .../qa/workunits/ceph-helpers.sh:119: teardown: stat -f -c %T . ../qa/workunits/ceph-helpers.sh:119: teardown: '[' ext2/ext3 == btrfs ']' ../qa/workunits/ceph-helpers.sh:122: teardown: rm -fr testdir/ceph-helpers ../qa/workunits/ceph-helpers.sh:1198: run_tests: for func in '$funcs' ../qa/workunits/ceph-helpers.sh:1199: run_tests: test_get_is_making_recovery_progress testdir/ceph-helpers ../qa/workunits/ceph-helpers.sh:844: test_get_is_making_recovery_progress: local dir=testdir/ceph-helpers ../qa/workunits/ceph-helpers.sh:846: test_get_is_making_recovery_progress: setup testdir/ceph-helpers ../qa/workunits/ceph-helpers.sh:92: setup: local dir=testdir/ceph-helpers ../qa/workunits/ceph-helpers.sh:93: setup: teardown testdir/ceph-helpers ../qa/workunits/ceph-helpers.sh:117: teardown: local dir=testdir/ceph-helpers ../qa/workunits/ceph-helpers.sh:118: teardown: kill_daemons testdir/ceph-helpers .../qa/workunits/ceph-helpers.sh:192: kill_daemons: shopt -q -o xtrace .../qa/workunits/ceph-helpers.sh:192: kill_daemons: echo true ../qa/workunits/ceph-helpers.sh:192: kill_daemons: local trace=true ../qa/workunits/ceph-helpers.sh:193: kill_daemons: true ../qa/workunits/ceph-helpers.sh:193: kill_daemons: shopt -u -o xtrace ../qa/workunits/ceph-helpers.sh:219: kill_daemons: return 0 .../qa/workunits/ceph-helpers.sh:119: teardown: stat -f -c %T . ../qa/workunits/ceph-helpers.sh:119: teardown: '[' ext2/ext3 == btrfs ']' ../qa/workunits/ceph-helpers.sh:122: teardown: rm -fr testdir/ceph-helpers ../qa/workunits/ceph-helpers.sh:94: setup: mkdir -p testdir/ceph-helpers ../qa/workunits/ceph-helpers.sh:847: test_get_is_making_recovery_progress: run_mon testdir/ceph-helpers a ../qa/workunits/ceph-helpers.sh:282: run_mon: local dir=testdir/ceph-helpers ../qa/workunits/ceph-helpers.sh:283: run_mon: shift ../qa/workunits/ceph-helpers.sh:284: run_mon: local id=a ../qa/workunits/ceph-helpers.sh:285: run_mon: shift ../qa/workunits/ceph-helpers.sh:286: run_mon: local data=testdir/ceph-helpers/a ../qa/workunits/ceph-helpers.sh:289: run_mon: ceph-mon --id a --mkfs --mon-data=testdir/ceph-helpers/a --run-dir=testdir/ceph-helpers ceph-mon: mon.noname-a 127.0.0.1:7109/0 is local, renaming to mon.a ceph-mon: set fsid to 3f1b7cce-0c19-4c66-990c-9bbd1a39c039 ceph-mon: created monfs at testdir/ceph-helpers/a for mon.a ../qa/workunits/ceph-helpers.sh:296: run_mon: ceph-mon --id a --mon-osd-full-ratio=.99 --mon-data-avail-crit=1 --paxos-propose-interval=0.1 --osd-crush-chooseleaf-type=0 --osd-pool-default-erasure-code-directory=.libs --debug-mon 20 --debug-ms 20 --debug-paxos 20 --chdir= --mon-data=testdir/ceph-helpers/a '--log-file=testdir/ceph-helpers/$name.log' '--admin-socket=testdir/ceph-helpers/$cluster-$name.asok' --mon-cluster-log-file=testdir/ceph-helpers/log --run-dir=testdir/ceph-helpers '--pid-file=testdir/ceph-helpers/$name.pid' ../qa/workunits/ceph-helpers.sh:314: run_mon: cat .../qa/workunits/ceph-helpers.sh:314: run_mon: get_config mon a fsid .../qa/workunits/ceph-helpers.sh:653: get_config: local daemon=mon .../qa/workunits/ceph-helpers.sh:654: get_config: local id=a .../qa/workunits/ceph-helpers.sh:655: get_config: local config=fsid .../qa/workunits/ceph-helpers.sh:657: get_config: CEPH_ARGS= .../qa/workunits/ceph-helpers.sh:657: get_config: ceph --format xml daemon testdir/ceph-helpers/ceph-mon.a.asok config get fsid .../qa/workunits/ceph-helpers.sh:660: get_config: xmlstarlet sel -t -m //fsid -v . -n .../qa/workunits/ceph-helpers.sh:314: run_mon: get_config mon a mon_host .../qa/workunits/ceph-helpers.sh:653: get_config: local daemon=mon .../qa/workunits/ceph-helpers.sh:654: get_config: local id=a .../qa/workunits/ceph-helpers.sh:655: get_config: local config=mon_host .../qa/workunits/ceph-helpers.sh:657: get_config: CEPH_ARGS= .../qa/workunits/ceph-helpers.sh:657: get_config: ceph --format xml daemon testdir/ceph-helpers/ceph-mon.a.asok config get mon_host .../qa/workunits/ceph-helpers.sh:660: get_config: xmlstarlet sel -t -m //mon_host -v . -n .../qa/workunits/ceph-helpers.sh:319: run_mon: get_config mon a mon_initial_members .../qa/workunits/ceph-helpers.sh:653: get_config: local daemon=mon .../qa/workunits/ceph-helpers.sh:654: get_config: local id=a .../qa/workunits/ceph-helpers.sh:655: get_config: local config=mon_initial_members .../qa/workunits/ceph-helpers.sh:657: get_config: CEPH_ARGS= .../qa/workunits/ceph-helpers.sh:657: get_config: ceph --format xml daemon testdir/ceph-helpers/ceph-mon.a.asok config get mon_initial_members .../qa/workunits/ceph-helpers.sh:660: get_config: xmlstarlet sel -t -m //mon_initial_members -v . -n ../qa/workunits/ceph-helpers.sh:319: run_mon: test -z '' ../qa/workunits/ceph-helpers.sh:320: run_mon: ceph osd pool delete rbd rbd --yes-i-really-really-mean-it *** DEVELOPER MODE: setting PATH, PYTHONPATH and LD_LIBRARY_PATH *** pool 'rbd' removed ../qa/workunits/ceph-helpers.sh:321: run_mon: ceph osd pool create rbd 4 *** DEVELOPER MODE: setting PATH, PYTHONPATH and LD_LIBRARY_PATH *** pool 'rbd' created ../qa/workunits/ceph-helpers.sh:848: test_get_is_making_recovery_progress: get_is_making_recovery_progress .../qa/workunits/ceph-helpers.sh:839: get_is_making_recovery_progress: ceph --format xml status .../qa/workunits/ceph-helpers.sh:839: get_is_making_recovery_progress: xmlstarlet sel -t -m //pgmap/recovering_keys_per_sec -v . -o ' ' -t -m //pgmap/recovering_bytes_per_sec -v . -o ' ' -t -m //pgmap/recovering_objects_per_sec -v . ../qa/workunits/ceph-helpers.sh:839: get_is_making_recovery_progress: local progress= ../qa/workunits/ceph-helpers.sh:840: get_is_making_recovery_progress: test -n '' ../qa/workunits/ceph-helpers.sh:849: test_get_is_making_recovery_progress: teardown testdir/ceph-helpers ../qa/workunits/ceph-helpers.sh:117: teardown: local dir=testdir/ceph-helpers ../qa/workunits/ceph-helpers.sh:118: teardown: kill_daemons testdir/ceph-helpers .../qa/workunits/ceph-helpers.sh:192: kill_daemons: shopt -q -o xtrace .../qa/workunits/ceph-helpers.sh:192: kill_daemons: echo true ../qa/workunits/ceph-helpers.sh:192: kill_daemons: local trace=true ../qa/workunits/ceph-helpers.sh:193: kill_daemons: true ../qa/workunits/ceph-helpers.sh:193: kill_daemons: shopt -u -o xtrace ../qa/workunits/ceph-helpers.sh:219: kill_daemons: return 0 .../qa/workunits/ceph-helpers.sh:119: teardown: stat -f -c %T . ../qa/workunits/ceph-helpers.sh:119: teardown: '[' ext2/ext3 == btrfs ']' ../qa/workunits/ceph-helpers.sh:122: teardown: rm -fr testdir/ceph-helpers ../qa/workunits/ceph-helpers.sh:1198: run_tests: for func in '$funcs' ../qa/workunits/ceph-helpers.sh:1199: run_tests: test_get_last_scrub_stamp testdir/ceph-helpers ../qa/workunits/ceph-helpers.sh:932: test_get_last_scrub_stamp: local dir=testdir/ceph-helpers ../qa/workunits/ceph-helpers.sh:934: test_get_last_scrub_stamp: setup testdir/ceph-helpers ../qa/workunits/ceph-helpers.sh:92: setup: local dir=testdir/ceph-helpers ../qa/workunits/ceph-helpers.sh:93: setup: teardown testdir/ceph-helpers ../qa/workunits/ceph-helpers.sh:117: teardown: local dir=testdir/ceph-helpers ../qa/workunits/ceph-helpers.sh:118: teardown: kill_daemons testdir/ceph-helpers .../qa/workunits/ceph-helpers.sh:192: kill_daemons: shopt -q -o xtrace .../qa/workunits/ceph-helpers.sh:192: kill_daemons: echo true ../qa/workunits/ceph-helpers.sh:192: kill_daemons: local trace=true ../qa/workunits/ceph-helpers.sh:193: kill_daemons: true ../qa/workunits/ceph-helpers.sh:193: kill_daemons: shopt -u -o xtrace ../qa/workunits/ceph-helpers.sh:219: kill_daemons: return 0 .../qa/workunits/ceph-helpers.sh:119: teardown: stat -f -c %T . ../qa/workunits/ceph-helpers.sh:119: teardown: '[' ext2/ext3 == btrfs ']' ../qa/workunits/ceph-helpers.sh:122: teardown: rm -fr testdir/ceph-helpers ../qa/workunits/ceph-helpers.sh:94: setup: mkdir -p testdir/ceph-helpers ../qa/workunits/ceph-helpers.sh:935: test_get_last_scrub_stamp: run_mon testdir/ceph-helpers a --osd_pool_default_size=1 ../qa/workunits/ceph-helpers.sh:282: run_mon: local dir=testdir/ceph-helpers ../qa/workunits/ceph-helpers.sh:283: run_mon: shift ../qa/workunits/ceph-helpers.sh:284: run_mon: local id=a ../qa/workunits/ceph-helpers.sh:285: run_mon: shift ../qa/workunits/ceph-helpers.sh:286: run_mon: local data=testdir/ceph-helpers/a ../qa/workunits/ceph-helpers.sh:289: run_mon: ceph-mon --id a --mkfs --mon-data=testdir/ceph-helpers/a --run-dir=testdir/ceph-helpers --osd_pool_default_size=1 ceph-mon: mon.noname-a 127.0.0.1:7109/0 is local, renaming to mon.a ceph-mon: set fsid to 3f1b7cce-0c19-4c66-990c-9bbd1a39c039 ceph-mon: created monfs at testdir/ceph-helpers/a for mon.a ../qa/workunits/ceph-helpers.sh:296: run_mon: ceph-mon --id a --mon-osd-full-ratio=.99 --mon-data-avail-crit=1 --paxos-propose-interval=0.1 --osd-crush-chooseleaf-type=0 --osd-pool-default-erasure-code-directory=.libs --debug-mon 20 --debug-ms 20 --debug-paxos 20 --chdir= --mon-data=testdir/ceph-helpers/a '--log-file=testdir/ceph-helpers/$name.log' '--admin-socket=testdir/ceph-helpers/$cluster-$name.asok' --mon-cluster-log-file=testdir/ceph-helpers/log --run-dir=testdir/ceph-helpers '--pid-file=testdir/ceph-helpers/$name.pid' --osd_pool_default_size=1 ../qa/workunits/ceph-helpers.sh:314: run_mon: cat .../qa/workunits/ceph-helpers.sh:314: run_mon: get_config mon a fsid .../qa/workunits/ceph-helpers.sh:653: get_config: local daemon=mon .../qa/workunits/ceph-helpers.sh:654: get_config: local id=a .../qa/workunits/ceph-helpers.sh:655: get_config: local config=fsid .../qa/workunits/ceph-helpers.sh:657: get_config: CEPH_ARGS= .../qa/workunits/ceph-helpers.sh:657: get_config: ceph --format xml daemon testdir/ceph-helpers/ceph-mon.a.asok config get fsid .../qa/workunits/ceph-helpers.sh:660: get_config: xmlstarlet sel -t -m //fsid -v . -n .../qa/workunits/ceph-helpers.sh:314: run_mon: get_config mon a mon_host .../qa/workunits/ceph-helpers.sh:653: get_config: local daemon=mon .../qa/workunits/ceph-helpers.sh:654: get_config: local id=a .../qa/workunits/ceph-helpers.sh:655: get_config: local config=mon_host .../qa/workunits/ceph-helpers.sh:657: get_config: CEPH_ARGS= .../qa/workunits/ceph-helpers.sh:657: get_config: ceph --format xml daemon testdir/ceph-helpers/ceph-mon.a.asok config get mon_host .../qa/workunits/ceph-helpers.sh:660: get_config: xmlstarlet sel -t -m //mon_host -v . -n .../qa/workunits/ceph-helpers.sh:319: run_mon: get_config mon a mon_initial_members .../qa/workunits/ceph-helpers.sh:653: get_config: local daemon=mon .../qa/workunits/ceph-helpers.sh:654: get_config: local id=a .../qa/workunits/ceph-helpers.sh:655: get_config: local config=mon_initial_members .../qa/workunits/ceph-helpers.sh:657: get_config: CEPH_ARGS= .../qa/workunits/ceph-helpers.sh:657: get_config: ceph --format xml daemon testdir/ceph-helpers/ceph-mon.a.asok config get mon_initial_members .../qa/workunits/ceph-helpers.sh:660: get_config: xmlstarlet sel -t -m //mon_initial_members -v . -n ../qa/workunits/ceph-helpers.sh:319: run_mon: test -z '' ../qa/workunits/ceph-helpers.sh:320: run_mon: ceph osd pool delete rbd rbd --yes-i-really-really-mean-it *** DEVELOPER MODE: setting PATH, PYTHONPATH and LD_LIBRARY_PATH *** pool 'rbd' removed ../qa/workunits/ceph-helpers.sh:321: run_mon: ceph osd pool create rbd 4 *** DEVELOPER MODE: setting PATH, PYTHONPATH and LD_LIBRARY_PATH *** pool 'rbd' created ../qa/workunits/ceph-helpers.sh:936: test_get_last_scrub_stamp: run_osd testdir/ceph-helpers 0 ../qa/workunits/ceph-helpers.sh:397: run_osd: local dir=testdir/ceph-helpers ../qa/workunits/ceph-helpers.sh:398: run_osd: shift ../qa/workunits/ceph-helpers.sh:399: run_osd: local id=0 ../qa/workunits/ceph-helpers.sh:400: run_osd: shift ../qa/workunits/ceph-helpers.sh:401: run_osd: local osd_data=testdir/ceph-helpers/0 ../qa/workunits/ceph-helpers.sh:403: run_osd: local ceph_disk_args ../qa/workunits/ceph-helpers.sh:404: run_osd: ceph_disk_args+=' --statedir=testdir/ceph-helpers' ../qa/workunits/ceph-helpers.sh:405: run_osd: ceph_disk_args+=' --sysconfdir=testdir/ceph-helpers' ../qa/workunits/ceph-helpers.sh:406: run_osd: ceph_disk_args+=' --prepend-to-path=' ../qa/workunits/ceph-helpers.sh:408: run_osd: mkdir -p testdir/ceph-helpers/0 ../qa/workunits/ceph-helpers.sh:409: run_osd: ceph-disk --statedir=testdir/ceph-helpers --sysconfdir=testdir/ceph-helpers --prepend-to-path= prepare testdir/ceph-helpers/0 2015-07-15 19:59:17.987425 7fd1627e57c0 -1 warning: unable to create /var/run/ceph: (13) Permission denied 2015-07-15 19:59:18.005157 7fd3d9c647c0 -1 warning: unable to create /var/run/ceph: (13) Permission denied 2015-07-15 19:59:18.019026 7f66c75f77c0 -1 warning: unable to create /var/run/ceph: (13) Permission denied ../qa/workunits/ceph-helpers.sh:412: run_osd: activate_osd testdir/ceph-helpers 0 ../qa/workunits/ceph-helpers.sh:479: activate_osd: local dir=testdir/ceph-helpers ../qa/workunits/ceph-helpers.sh:480: activate_osd: shift ../qa/workunits/ceph-helpers.sh:481: activate_osd: local id=0 ../qa/workunits/ceph-helpers.sh:482: activate_osd: shift ../qa/workunits/ceph-helpers.sh:483: activate_osd: local osd_data=testdir/ceph-helpers/0 ../qa/workunits/ceph-helpers.sh:485: activate_osd: local ceph_disk_args ../qa/workunits/ceph-helpers.sh:486: activate_osd: ceph_disk_args+=' --statedir=testdir/ceph-helpers' ../qa/workunits/ceph-helpers.sh:487: activate_osd: ceph_disk_args+=' --sysconfdir=testdir/ceph-helpers' ../qa/workunits/ceph-helpers.sh:488: activate_osd: ceph_disk_args+=' --prepend-to-path=' ../qa/workunits/ceph-helpers.sh:490: activate_osd: local 'ceph_args=--fsid=3f1b7cce-0c19-4c66-990c-9bbd1a39c039 --auth-supported=none --mon-host=127.0.0.1:7109 ' ../qa/workunits/ceph-helpers.sh:491: activate_osd: ceph_args+=' --osd-backfill-full-ratio=.99' ../qa/workunits/ceph-helpers.sh:492: activate_osd: ceph_args+=' --osd-failsafe-full-ratio=.99' ../qa/workunits/ceph-helpers.sh:493: activate_osd: ceph_args+=' --osd-journal-size=100' ../qa/workunits/ceph-helpers.sh:494: activate_osd: ceph_args+=' --osd-data=testdir/ceph-helpers/0' ../qa/workunits/ceph-helpers.sh:495: activate_osd: ceph_args+=' --chdir=' ../qa/workunits/ceph-helpers.sh:496: activate_osd: ceph_args+=' --osd-pool-default-erasure-code-directory=.libs' ../qa/workunits/ceph-helpers.sh:497: activate_osd: ceph_args+=' --osd-class-dir=.libs' ../qa/workunits/ceph-helpers.sh:498: activate_osd: ceph_args+=' --run-dir=testdir/ceph-helpers' ../qa/workunits/ceph-helpers.sh:499: activate_osd: ceph_args+=' --debug-osd=20' ../qa/workunits/ceph-helpers.sh:500: activate_osd: ceph_args+=' --log-file=testdir/ceph-helpers/$name.log' ../qa/workunits/ceph-helpers.sh:501: activate_osd: ceph_args+=' --pid-file=testdir/ceph-helpers/$name.pid' ../qa/workunits/ceph-helpers.sh:502: activate_osd: ceph_args+=' ' ../qa/workunits/ceph-helpers.sh:503: activate_osd: ceph_args+= ../qa/workunits/ceph-helpers.sh:504: activate_osd: mkdir -p testdir/ceph-helpers/0 ../qa/workunits/ceph-helpers.sh:505: activate_osd: CEPH_ARGS='--fsid=3f1b7cce-0c19-4c66-990c-9bbd1a39c039 --auth-supported=none --mon-host=127.0.0.1:7109 --osd-backfill-full-ratio=.99 --osd-failsafe-full-ratio=.99 --osd-journal-size=100 --osd-data=testdir/ceph-helpers/0 --chdir= --osd-pool-default-erasure-code-directory=.libs --osd-class-dir=.libs --run-dir=testdir/ceph-helpers --debug-osd=20 --log-file=testdir/ceph-helpers/$name.log --pid-file=testdir/ceph-helpers/$name.pid ' ../qa/workunits/ceph-helpers.sh:505: activate_osd: ceph-disk --statedir=testdir/ceph-helpers --sysconfdir=testdir/ceph-helpers --prepend-to-path= activate --mark-init=none testdir/ceph-helpers/0 *** DEVELOPER MODE: setting PATH, PYTHONPATH and LD_LIBRARY_PATH *** *** DEVELOPER MODE: setting PATH, PYTHONPATH and LD_LIBRARY_PATH *** got monmap epoch 1 2015-07-15 19:59:20.397852 7fda274e67c0 -1 journal FileJournal::_open: disabling aio for non-block journal. Use journal_force_aio to force use of aio anyway 2015-07-15 19:59:20.504640 7fda274e67c0 -1 journal FileJournal::_open: disabling aio for non-block journal. Use journal_force_aio to force use of aio anyway 2015-07-15 19:59:20.505120 7fda274e67c0 -1 filestore(testdir/ceph-helpers/0) could not find -1/23c2fcde/osd_superblock/0 in index: (2) No such file or directory 2015-07-15 19:59:20.605927 7fda274e67c0 -1 created object store testdir/ceph-helpers/0 journal testdir/ceph-helpers/0/journal for osd.0 fsid 3f1b7cce-0c19-4c66-990c-9bbd1a39c039 2015-07-15 19:59:20.605993 7fda274e67c0 -1 auth: error reading file: testdir/ceph-helpers/0/keyring: can't open testdir/ceph-helpers/0/keyring: (2) No such file or directory 2015-07-15 19:59:20.606171 7fda274e67c0 -1 created new key in keyring testdir/ceph-helpers/0/keyring *** DEVELOPER MODE: setting PATH, PYTHONPATH and LD_LIBRARY_PATH *** added key for osd.0 starting osd.0 at :/0 osd_data testdir/ceph-helpers/0 testdir/ceph-helpers/0/journal .../qa/workunits/ceph-helpers.sh:510: activate_osd: cat testdir/ceph-helpers/0/whoami ../qa/workunits/ceph-helpers.sh:510: activate_osd: '[' 0 = 0 ']' ../qa/workunits/ceph-helpers.sh:512: activate_osd: ceph osd crush create-or-move 0 1 root=default host=localhost *** DEVELOPER MODE: setting PATH, PYTHONPATH and LD_LIBRARY_PATH *** create-or-move updating item name 'osd.0' weight 1 at location {host=localhost,root=default} to crush map ../qa/workunits/ceph-helpers.sh:514: activate_osd: wait_for_osd up 0 ../qa/workunits/ceph-helpers.sh:550: wait_for_osd: local state=up ../qa/workunits/ceph-helpers.sh:551: wait_for_osd: local id=0 ../qa/workunits/ceph-helpers.sh:553: wait_for_osd: status=1 ../qa/workunits/ceph-helpers.sh:554: wait_for_osd: (( i=0 )) ../qa/workunits/ceph-helpers.sh:554: wait_for_osd: (( i < 120 )) ../qa/workunits/ceph-helpers.sh:555: wait_for_osd: ceph osd dump ../qa/workunits/ceph-helpers.sh:555: wait_for_osd: grep 'osd.0 up' *** DEVELOPER MODE: setting PATH, PYTHONPATH and LD_LIBRARY_PATH *** osd.0 up in weight 1 up_from 5 up_thru 6 down_at 0 last_clean_interval [0,0) 127.0.0.1:6804/29933 127.0.0.1:6805/29933 127.0.0.1:6806/29933 127.0.0.1:6807/29933 exists,up d0bf3a6f-f6c7-47d6-be01-3b66da6c1aed ../qa/workunits/ceph-helpers.sh:558: wait_for_osd: status=0 ../qa/workunits/ceph-helpers.sh:559: wait_for_osd: break ../qa/workunits/ceph-helpers.sh:562: wait_for_osd: return 0 ../qa/workunits/ceph-helpers.sh:937: test_get_last_scrub_stamp: wait_for_clean ../qa/workunits/ceph-helpers.sh:981: wait_for_clean: local status=1 .../qa/workunits/ceph-helpers.sh:982: wait_for_clean: get_num_active_clean .../qa/workunits/ceph-helpers.sh:863: get_num_active_clean: local 'expression=(' .../qa/workunits/ceph-helpers.sh:864: get_num_active_clean: expression+='contains(.,'\''active'\'') and ' .../qa/workunits/ceph-helpers.sh:865: get_num_active_clean: expression+='contains(.,'\''clean'\'') and ' .../qa/workunits/ceph-helpers.sh:866: get_num_active_clean: expression+='not(contains(.,'\''stale'\''))' .../qa/workunits/ceph-helpers.sh:867: get_num_active_clean: expression+=')' .../qa/workunits/ceph-helpers.sh:871: get_num_active_clean: ceph --format xml pg dump pgs .../qa/workunits/ceph-helpers.sh:872: get_num_active_clean: xmlstarlet sel -t -m '//pg_stat/state[(contains(.,'\''active'\'') and contains(.,'\''clean'\'') and not(contains(.,'\''stale'\'')))]' -v . -n .../qa/workunits/ceph-helpers.sh:873: get_num_active_clean: grep -v '^$' .../qa/workunits/ceph-helpers.sh:873: get_num_active_clean: wc -l ../qa/workunits/ceph-helpers.sh:982: wait_for_clean: local num_active_clean=0 ../qa/workunits/ceph-helpers.sh:983: wait_for_clean: local cur_active_clean ../qa/workunits/ceph-helpers.sh:984: wait_for_clean: local -i timer=0 ../qa/workunits/ceph-helpers.sh:985: wait_for_clean: is_clean .../qa/workunits/ceph-helpers.sh:952: is_clean: get_num_pgs .../qa/workunits/ceph-helpers.sh:898: get_num_pgs: ceph --format xml status .../qa/workunits/ceph-helpers.sh:899: get_num_pgs: xmlstarlet sel -t -m //pgmap/num_pgs -v . ../qa/workunits/ceph-helpers.sh:952: is_clean: num_pgs=4 ../qa/workunits/ceph-helpers.sh:953: is_clean: test 4 '!=' 0 .../qa/workunits/ceph-helpers.sh:954: is_clean: get_num_active_clean .../qa/workunits/ceph-helpers.sh:863: get_num_active_clean: local 'expression=(' .../qa/workunits/ceph-helpers.sh:864: get_num_active_clean: expression+='contains(.,'\''active'\'') and ' .../qa/workunits/ceph-helpers.sh:865: get_num_active_clean: expression+='contains(.,'\''clean'\'') and ' .../qa/workunits/ceph-helpers.sh:866: get_num_active_clean: expression+='not(contains(.,'\''stale'\''))' .../qa/workunits/ceph-helpers.sh:867: get_num_active_clean: expression+=')' .../qa/workunits/ceph-helpers.sh:871: get_num_active_clean: ceph --format xml pg dump pgs .../qa/workunits/ceph-helpers.sh:872: get_num_active_clean: xmlstarlet sel -t -m '//pg_stat/state[(contains(.,'\''active'\'') and contains(.,'\''clean'\'') and not(contains(.,'\''stale'\'')))]' -v . -n .../qa/workunits/ceph-helpers.sh:873: get_num_active_clean: grep -v '^$' .../qa/workunits/ceph-helpers.sh:873: get_num_active_clean: wc -l ../qa/workunits/ceph-helpers.sh:954: is_clean: test 0 = 4 ../qa/workunits/ceph-helpers.sh:954: is_clean: return 1 ../qa/workunits/ceph-helpers.sh:986: wait_for_clean: get_is_making_recovery_progress .../qa/workunits/ceph-helpers.sh:839: get_is_making_recovery_progress: ceph --format xml status .../qa/workunits/ceph-helpers.sh:839: get_is_making_recovery_progress: xmlstarlet sel -t -m //pgmap/recovering_keys_per_sec -v . -o ' ' -t -m //pgmap/recovering_bytes_per_sec -v . -o ' ' -t -m //pgmap/recovering_objects_per_sec -v . ../qa/workunits/ceph-helpers.sh:839: get_is_making_recovery_progress: local progress= ../qa/workunits/ceph-helpers.sh:840: get_is_making_recovery_progress: test -n '' ../qa/workunits/ceph-helpers.sh:988: wait_for_clean: (( timer >= 120 )) .../qa/workunits/ceph-helpers.sh:993: wait_for_clean: get_num_active_clean .../qa/workunits/ceph-helpers.sh:863: get_num_active_clean: local 'expression=(' .../qa/workunits/ceph-helpers.sh:864: get_num_active_clean: expression+='contains(.,'\''active'\'') and ' .../qa/workunits/ceph-helpers.sh:865: get_num_active_clean: expression+='contains(.,'\''clean'\'') and ' .../qa/workunits/ceph-helpers.sh:866: get_num_active_clean: expression+='not(contains(.,'\''stale'\''))' .../qa/workunits/ceph-helpers.sh:867: get_num_active_clean: expression+=')' .../qa/workunits/ceph-helpers.sh:871: get_num_active_clean: ceph --format xml pg dump pgs .../qa/workunits/ceph-helpers.sh:872: get_num_active_clean: xmlstarlet sel -t -m '//pg_stat/state[(contains(.,'\''active'\'') and contains(.,'\''clean'\'') and not(contains(.,'\''stale'\'')))]' -v . -n .../qa/workunits/ceph-helpers.sh:873: get_num_active_clean: wc -l .../qa/workunits/ceph-helpers.sh:873: get_num_active_clean: grep -v '^$' ../qa/workunits/ceph-helpers.sh:993: wait_for_clean: cur_active_clean=0 ../qa/workunits/ceph-helpers.sh:994: wait_for_clean: test 0 '!=' 0 ../qa/workunits/ceph-helpers.sh:998: wait_for_clean: sleep 1 .../qa/workunits/ceph-helpers.sh:999: wait_for_clean: expr 0 + 1 ../qa/workunits/ceph-helpers.sh:999: wait_for_clean: timer=1 ../qa/workunits/ceph-helpers.sh:985: wait_for_clean: is_clean .../qa/workunits/ceph-helpers.sh:952: is_clean: get_num_pgs .../qa/workunits/ceph-helpers.sh:898: get_num_pgs: ceph --format xml status .../qa/workunits/ceph-helpers.sh:899: get_num_pgs: xmlstarlet sel -t -m //pgmap/num_pgs -v . ../qa/workunits/ceph-helpers.sh:952: is_clean: num_pgs=4 ../qa/workunits/ceph-helpers.sh:953: is_clean: test 4 '!=' 0 .../qa/workunits/ceph-helpers.sh:954: is_clean: get_num_active_clean .../qa/workunits/ceph-helpers.sh:863: get_num_active_clean: local 'expression=(' .../qa/workunits/ceph-helpers.sh:864: get_num_active_clean: expression+='contains(.,'\''active'\'') and ' .../qa/workunits/ceph-helpers.sh:865: get_num_active_clean: expression+='contains(.,'\''clean'\'') and ' .../qa/workunits/ceph-helpers.sh:866: get_num_active_clean: expression+='not(contains(.,'\''stale'\''))' .../qa/workunits/ceph-helpers.sh:867: get_num_active_clean: expression+=')' .../qa/workunits/ceph-helpers.sh:871: get_num_active_clean: ceph --format xml pg dump pgs .../qa/workunits/ceph-helpers.sh:872: get_num_active_clean: xmlstarlet sel -t -m '//pg_stat/state[(contains(.,'\''active'\'') and contains(.,'\''clean'\'') and not(contains(.,'\''stale'\'')))]' -v . -n .../qa/workunits/ceph-helpers.sh:873: get_num_active_clean: grep -v '^$' .../qa/workunits/ceph-helpers.sh:873: get_num_active_clean: wc -l ../qa/workunits/ceph-helpers.sh:954: is_clean: test 0 = 4 ../qa/workunits/ceph-helpers.sh:954: is_clean: return 1 ../qa/workunits/ceph-helpers.sh:986: wait_for_clean: get_is_making_recovery_progress .../qa/workunits/ceph-helpers.sh:839: get_is_making_recovery_progress: ceph --format xml status .../qa/workunits/ceph-helpers.sh:839: get_is_making_recovery_progress: xmlstarlet sel -t -m //pgmap/recovering_keys_per_sec -v . -o ' ' -t -m //pgmap/recovering_bytes_per_sec -v . -o ' ' -t -m //pgmap/recovering_objects_per_sec -v . ../qa/workunits/ceph-helpers.sh:839: get_is_making_recovery_progress: local progress= ../qa/workunits/ceph-helpers.sh:840: get_is_making_recovery_progress: test -n '' ../qa/workunits/ceph-helpers.sh:988: wait_for_clean: (( timer >= 120 )) .../qa/workunits/ceph-helpers.sh:993: wait_for_clean: get_num_active_clean .../qa/workunits/ceph-helpers.sh:863: get_num_active_clean: local 'expression=(' .../qa/workunits/ceph-helpers.sh:864: get_num_active_clean: expression+='contains(.,'\''active'\'') and ' .../qa/workunits/ceph-helpers.sh:865: get_num_active_clean: expression+='contains(.,'\''clean'\'') and ' .../qa/workunits/ceph-helpers.sh:866: get_num_active_clean: expression+='not(contains(.,'\''stale'\''))' .../qa/workunits/ceph-helpers.sh:867: get_num_active_clean: expression+=')' .../qa/workunits/ceph-helpers.sh:871: get_num_active_clean: ceph --format xml pg dump pgs .../qa/workunits/ceph-helpers.sh:872: get_num_active_clean: xmlstarlet sel -t -m '//pg_stat/state[(contains(.,'\''active'\'') and contains(.,'\''clean'\'') and not(contains(.,'\''stale'\'')))]' -v . -n .../qa/workunits/ceph-helpers.sh:873: get_num_active_clean: grep -v '^$' .../qa/workunits/ceph-helpers.sh:873: get_num_active_clean: wc -l ../qa/workunits/ceph-helpers.sh:993: wait_for_clean: cur_active_clean=0 ../qa/workunits/ceph-helpers.sh:994: wait_for_clean: test 0 '!=' 0 ../qa/workunits/ceph-helpers.sh:998: wait_for_clean: sleep 1 .../qa/workunits/ceph-helpers.sh:999: wait_for_clean: expr 1 + 1 ../qa/workunits/ceph-helpers.sh:999: wait_for_clean: timer=2 ../qa/workunits/ceph-helpers.sh:985: wait_for_clean: is_clean .../qa/workunits/ceph-helpers.sh:952: is_clean: get_num_pgs .../qa/workunits/ceph-helpers.sh:898: get_num_pgs: ceph --format xml status .../qa/workunits/ceph-helpers.sh:899: get_num_pgs: xmlstarlet sel -t -m //pgmap/num_pgs -v . ../qa/workunits/ceph-helpers.sh:952: is_clean: num_pgs=4 ../qa/workunits/ceph-helpers.sh:953: is_clean: test 4 '!=' 0 .../qa/workunits/ceph-helpers.sh:954: is_clean: get_num_active_clean .../qa/workunits/ceph-helpers.sh:863: get_num_active_clean: local 'expression=(' .../qa/workunits/ceph-helpers.sh:864: get_num_active_clean: expression+='contains(.,'\''active'\'') and ' .../qa/workunits/ceph-helpers.sh:865: get_num_active_clean: expression+='contains(.,'\''clean'\'') and ' .../qa/workunits/ceph-helpers.sh:866: get_num_active_clean: expression+='not(contains(.,'\''stale'\''))' .../qa/workunits/ceph-helpers.sh:867: get_num_active_clean: expression+=')' .../qa/workunits/ceph-helpers.sh:871: get_num_active_clean: ceph --format xml pg dump pgs .../qa/workunits/ceph-helpers.sh:872: get_num_active_clean: xmlstarlet sel -t -m '//pg_stat/state[(contains(.,'\''active'\'') and contains(.,'\''clean'\'') and not(contains(.,'\''stale'\'')))]' -v . -n .../qa/workunits/ceph-helpers.sh:873: get_num_active_clean: grep -v '^$' .../qa/workunits/ceph-helpers.sh:873: get_num_active_clean: wc -l ../qa/workunits/ceph-helpers.sh:954: is_clean: test 0 = 4 ../qa/workunits/ceph-helpers.sh:954: is_clean: return 1 ../qa/workunits/ceph-helpers.sh:986: wait_for_clean: get_is_making_recovery_progress .../qa/workunits/ceph-helpers.sh:839: get_is_making_recovery_progress: ceph --format xml status .../qa/workunits/ceph-helpers.sh:839: get_is_making_recovery_progress: xmlstarlet sel -t -m //pgmap/recovering_keys_per_sec -v . -o ' ' -t -m //pgmap/recovering_bytes_per_sec -v . -o ' ' -t -m //pgmap/recovering_objects_per_sec -v . ../qa/workunits/ceph-helpers.sh:839: get_is_making_recovery_progress: local progress= ../qa/workunits/ceph-helpers.sh:840: get_is_making_recovery_progress: test -n '' ../qa/workunits/ceph-helpers.sh:988: wait_for_clean: (( timer >= 120 )) .../qa/workunits/ceph-helpers.sh:993: wait_for_clean: get_num_active_clean .../qa/workunits/ceph-helpers.sh:863: get_num_active_clean: local 'expression=(' .../qa/workunits/ceph-helpers.sh:864: get_num_active_clean: expression+='contains(.,'\''active'\'') and ' .../qa/workunits/ceph-helpers.sh:865: get_num_active_clean: expression+='contains(.,'\''clean'\'') and ' .../qa/workunits/ceph-helpers.sh:866: get_num_active_clean: expression+='not(contains(.,'\''stale'\''))' .../qa/workunits/ceph-helpers.sh:867: get_num_active_clean: expression+=')' .../qa/workunits/ceph-helpers.sh:871: get_num_active_clean: ceph --format xml pg dump pgs .../qa/workunits/ceph-helpers.sh:872: get_num_active_clean: xmlstarlet sel -t -m '//pg_stat/state[(contains(.,'\''active'\'') and contains(.,'\''clean'\'') and not(contains(.,'\''stale'\'')))]' -v . -n .../qa/workunits/ceph-helpers.sh:873: get_num_active_clean: grep -v '^$' .../qa/workunits/ceph-helpers.sh:873: get_num_active_clean: wc -l ../qa/workunits/ceph-helpers.sh:993: wait_for_clean: cur_active_clean=0 ../qa/workunits/ceph-helpers.sh:994: wait_for_clean: test 0 '!=' 0 ../qa/workunits/ceph-helpers.sh:998: wait_for_clean: sleep 1 .../qa/workunits/ceph-helpers.sh:999: wait_for_clean: expr 2 + 1 ../qa/workunits/ceph-helpers.sh:999: wait_for_clean: timer=3 ../qa/workunits/ceph-helpers.sh:985: wait_for_clean: is_clean .../qa/workunits/ceph-helpers.sh:952: is_clean: get_num_pgs .../qa/workunits/ceph-helpers.sh:898: get_num_pgs: ceph --format xml status .../qa/workunits/ceph-helpers.sh:899: get_num_pgs: xmlstarlet sel -t -m //pgmap/num_pgs -v . ../qa/workunits/ceph-helpers.sh:952: is_clean: num_pgs=4 ../qa/workunits/ceph-helpers.sh:953: is_clean: test 4 '!=' 0 .../qa/workunits/ceph-helpers.sh:954: is_clean: get_num_active_clean .../qa/workunits/ceph-helpers.sh:863: get_num_active_clean: local 'expression=(' .../qa/workunits/ceph-helpers.sh:864: get_num_active_clean: expression+='contains(.,'\''active'\'') and ' .../qa/workunits/ceph-helpers.sh:865: get_num_active_clean: expression+='contains(.,'\''clean'\'') and ' .../qa/workunits/ceph-helpers.sh:866: get_num_active_clean: expression+='not(contains(.,'\''stale'\''))' .../qa/workunits/ceph-helpers.sh:867: get_num_active_clean: expression+=')' .../qa/workunits/ceph-helpers.sh:871: get_num_active_clean: ceph --format xml pg dump pgs .../qa/workunits/ceph-helpers.sh:872: get_num_active_clean: xmlstarlet sel -t -m '//pg_stat/state[(contains(.,'\''active'\'') and contains(.,'\''clean'\'') and not(contains(.,'\''stale'\'')))]' -v . -n .../qa/workunits/ceph-helpers.sh:873: get_num_active_clean: grep -v '^$' .../qa/workunits/ceph-helpers.sh:873: get_num_active_clean: wc -l ../qa/workunits/ceph-helpers.sh:954: is_clean: test 0 = 4 ../qa/workunits/ceph-helpers.sh:954: is_clean: return 1 ../qa/workunits/ceph-helpers.sh:986: wait_for_clean: get_is_making_recovery_progress .../qa/workunits/ceph-helpers.sh:839: get_is_making_recovery_progress: ceph --format xml status .../qa/workunits/ceph-helpers.sh:839: get_is_making_recovery_progress: xmlstarlet sel -t -m //pgmap/recovering_keys_per_sec -v . -o ' ' -t -m //pgmap/recovering_bytes_per_sec -v . -o ' ' -t -m //pgmap/recovering_objects_per_sec -v . ../qa/workunits/ceph-helpers.sh:839: get_is_making_recovery_progress: local progress= ../qa/workunits/ceph-helpers.sh:840: get_is_making_recovery_progress: test -n '' ../qa/workunits/ceph-helpers.sh:988: wait_for_clean: (( timer >= 120 )) .../qa/workunits/ceph-helpers.sh:993: wait_for_clean: get_num_active_clean .../qa/workunits/ceph-helpers.sh:863: get_num_active_clean: local 'expression=(' .../qa/workunits/ceph-helpers.sh:864: get_num_active_clean: expression+='contains(.,'\''active'\'') and ' .../qa/workunits/ceph-helpers.sh:865: get_num_active_clean: expression+='contains(.,'\''clean'\'') and ' .../qa/workunits/ceph-helpers.sh:866: get_num_active_clean: expression+='not(contains(.,'\''stale'\''))' .../qa/workunits/ceph-helpers.sh:867: get_num_active_clean: expression+=')' .../qa/workunits/ceph-helpers.sh:871: get_num_active_clean: ceph --format xml pg dump pgs .../qa/workunits/ceph-helpers.sh:872: get_num_active_clean: xmlstarlet sel -t -m '//pg_stat/state[(contains(.,'\''active'\'') and contains(.,'\''clean'\'') and not(contains(.,'\''stale'\'')))]' -v . -n .../qa/workunits/ceph-helpers.sh:873: get_num_active_clean: grep -v '^$' .../qa/workunits/ceph-helpers.sh:873: get_num_active_clean: wc -l ../qa/workunits/ceph-helpers.sh:993: wait_for_clean: cur_active_clean=0 ../qa/workunits/ceph-helpers.sh:994: wait_for_clean: test 0 '!=' 0 ../qa/workunits/ceph-helpers.sh:998: wait_for_clean: sleep 1 .../qa/workunits/ceph-helpers.sh:999: wait_for_clean: expr 3 + 1 ../qa/workunits/ceph-helpers.sh:999: wait_for_clean: timer=4 ../qa/workunits/ceph-helpers.sh:985: wait_for_clean: is_clean .../qa/workunits/ceph-helpers.sh:952: is_clean: get_num_pgs .../qa/workunits/ceph-helpers.sh:898: get_num_pgs: ceph --format xml status .../qa/workunits/ceph-helpers.sh:899: get_num_pgs: xmlstarlet sel -t -m //pgmap/num_pgs -v . ../qa/workunits/ceph-helpers.sh:952: is_clean: num_pgs=4 ../qa/workunits/ceph-helpers.sh:953: is_clean: test 4 '!=' 0 .../qa/workunits/ceph-helpers.sh:954: is_clean: get_num_active_clean .../qa/workunits/ceph-helpers.sh:863: get_num_active_clean: local 'expression=(' .../qa/workunits/ceph-helpers.sh:864: get_num_active_clean: expression+='contains(.,'\''active'\'') and ' .../qa/workunits/ceph-helpers.sh:865: get_num_active_clean: expression+='contains(.,'\''clean'\'') and ' .../qa/workunits/ceph-helpers.sh:866: get_num_active_clean: expression+='not(contains(.,'\''stale'\''))' .../qa/workunits/ceph-helpers.sh:867: get_num_active_clean: expression+=')' .../qa/workunits/ceph-helpers.sh:871: get_num_active_clean: ceph --format xml pg dump pgs .../qa/workunits/ceph-helpers.sh:872: get_num_active_clean: xmlstarlet sel -t -m '//pg_stat/state[(contains(.,'\''active'\'') and contains(.,'\''clean'\'') and not(contains(.,'\''stale'\'')))]' -v . -n .../qa/workunits/ceph-helpers.sh:873: get_num_active_clean: grep -v '^$' .../qa/workunits/ceph-helpers.sh:873: get_num_active_clean: wc -l ../qa/workunits/ceph-helpers.sh:954: is_clean: test 0 = 4 ../qa/workunits/ceph-helpers.sh:954: is_clean: return 1 ../qa/workunits/ceph-helpers.sh:986: wait_for_clean: get_is_making_recovery_progress .../qa/workunits/ceph-helpers.sh:839: get_is_making_recovery_progress: ceph --format xml status .../qa/workunits/ceph-helpers.sh:839: get_is_making_recovery_progress: xmlstarlet sel -t -m //pgmap/recovering_keys_per_sec -v . -o ' ' -t -m //pgmap/recovering_bytes_per_sec -v . -o ' ' -t -m //pgmap/recovering_objects_per_sec -v . ../qa/workunits/ceph-helpers.sh:839: get_is_making_recovery_progress: local progress= ../qa/workunits/ceph-helpers.sh:840: get_is_making_recovery_progress: test -n '' ../qa/workunits/ceph-helpers.sh:988: wait_for_clean: (( timer >= 120 )) .../qa/workunits/ceph-helpers.sh:993: wait_for_clean: get_num_active_clean .../qa/workunits/ceph-helpers.sh:863: get_num_active_clean: local 'expression=(' .../qa/workunits/ceph-helpers.sh:864: get_num_active_clean: expression+='contains(.,'\''active'\'') and ' .../qa/workunits/ceph-helpers.sh:865: get_num_active_clean: expression+='contains(.,'\''clean'\'') and ' .../qa/workunits/ceph-helpers.sh:866: get_num_active_clean: expression+='not(contains(.,'\''stale'\''))' .../qa/workunits/ceph-helpers.sh:867: get_num_active_clean: expression+=')' .../qa/workunits/ceph-helpers.sh:871: get_num_active_clean: ceph --format xml pg dump pgs .../qa/workunits/ceph-helpers.sh:872: get_num_active_clean: xmlstarlet sel -t -m '//pg_stat/state[(contains(.,'\''active'\'') and contains(.,'\''clean'\'') and not(contains(.,'\''stale'\'')))]' -v . -n .../qa/workunits/ceph-helpers.sh:873: get_num_active_clean: grep -v '^$' .../qa/workunits/ceph-helpers.sh:873: get_num_active_clean: wc -l ../qa/workunits/ceph-helpers.sh:993: wait_for_clean: cur_active_clean=0 ../qa/workunits/ceph-helpers.sh:994: wait_for_clean: test 0 '!=' 0 ../qa/workunits/ceph-helpers.sh:998: wait_for_clean: sleep 1 .../qa/workunits/ceph-helpers.sh:999: wait_for_clean: expr 4 + 1 ../qa/workunits/ceph-helpers.sh:999: wait_for_clean: timer=5 ../qa/workunits/ceph-helpers.sh:985: wait_for_clean: is_clean .../qa/workunits/ceph-helpers.sh:952: is_clean: get_num_pgs .../qa/workunits/ceph-helpers.sh:898: get_num_pgs: ceph --format xml status .../qa/workunits/ceph-helpers.sh:899: get_num_pgs: xmlstarlet sel -t -m //pgmap/num_pgs -v . ../qa/workunits/ceph-helpers.sh:952: is_clean: num_pgs=4 ../qa/workunits/ceph-helpers.sh:953: is_clean: test 4 '!=' 0 .../qa/workunits/ceph-helpers.sh:954: is_clean: get_num_active_clean .../qa/workunits/ceph-helpers.sh:863: get_num_active_clean: local 'expression=(' .../qa/workunits/ceph-helpers.sh:864: get_num_active_clean: expression+='contains(.,'\''active'\'') and ' .../qa/workunits/ceph-helpers.sh:865: get_num_active_clean: expression+='contains(.,'\''clean'\'') and ' .../qa/workunits/ceph-helpers.sh:866: get_num_active_clean: expression+='not(contains(.,'\''stale'\''))' .../qa/workunits/ceph-helpers.sh:867: get_num_active_clean: expression+=')' .../qa/workunits/ceph-helpers.sh:871: get_num_active_clean: ceph --format xml pg dump pgs .../qa/workunits/ceph-helpers.sh:872: get_num_active_clean: xmlstarlet sel -t -m '//pg_stat/state[(contains(.,'\''active'\'') and contains(.,'\''clean'\'') and not(contains(.,'\''stale'\'')))]' -v . -n .../qa/workunits/ceph-helpers.sh:873: get_num_active_clean: wc -l .../qa/workunits/ceph-helpers.sh:873: get_num_active_clean: grep -v '^$' ../qa/workunits/ceph-helpers.sh:954: is_clean: test 0 = 4 ../qa/workunits/ceph-helpers.sh:954: is_clean: return 1 ../qa/workunits/ceph-helpers.sh:986: wait_for_clean: get_is_making_recovery_progress .../qa/workunits/ceph-helpers.sh:839: get_is_making_recovery_progress: ceph --format xml status .../qa/workunits/ceph-helpers.sh:839: get_is_making_recovery_progress: xmlstarlet sel -t -m //pgmap/recovering_keys_per_sec -v . -o ' ' -t -m //pgmap/recovering_bytes_per_sec -v . -o ' ' -t -m //pgmap/recovering_objects_per_sec -v . ../qa/workunits/ceph-helpers.sh:839: get_is_making_recovery_progress: local progress= ../qa/workunits/ceph-helpers.sh:840: get_is_making_recovery_progress: test -n '' ../qa/workunits/ceph-helpers.sh:988: wait_for_clean: (( timer >= 120 )) .../qa/workunits/ceph-helpers.sh:993: wait_for_clean: get_num_active_clean .../qa/workunits/ceph-helpers.sh:863: get_num_active_clean: local 'expression=(' .../qa/workunits/ceph-helpers.sh:864: get_num_active_clean: expression+='contains(.,'\''active'\'') and ' .../qa/workunits/ceph-helpers.sh:865: get_num_active_clean: expression+='contains(.,'\''clean'\'') and ' .../qa/workunits/ceph-helpers.sh:866: get_num_active_clean: expression+='not(contains(.,'\''stale'\''))' .../qa/workunits/ceph-helpers.sh:867: get_num_active_clean: expression+=')' .../qa/workunits/ceph-helpers.sh:871: get_num_active_clean: ceph --format xml pg dump pgs .../qa/workunits/ceph-helpers.sh:872: get_num_active_clean: xmlstarlet sel -t -m '//pg_stat/state[(contains(.,'\''active'\'') and contains(.,'\''clean'\'') and not(contains(.,'\''stale'\'')))]' -v . -n .../qa/workunits/ceph-helpers.sh:873: get_num_active_clean: grep -v '^$' .../qa/workunits/ceph-helpers.sh:873: get_num_active_clean: wc -l ../qa/workunits/ceph-helpers.sh:993: wait_for_clean: cur_active_clean=0 ../qa/workunits/ceph-helpers.sh:994: wait_for_clean: test 0 '!=' 0 ../qa/workunits/ceph-helpers.sh:998: wait_for_clean: sleep 1 .../qa/workunits/ceph-helpers.sh:999: wait_for_clean: expr 5 + 1 ../qa/workunits/ceph-helpers.sh:999: wait_for_clean: timer=6 ../qa/workunits/ceph-helpers.sh:985: wait_for_clean: is_clean .../qa/workunits/ceph-helpers.sh:952: is_clean: get_num_pgs .../qa/workunits/ceph-helpers.sh:898: get_num_pgs: ceph --format xml status .../qa/workunits/ceph-helpers.sh:899: get_num_pgs: xmlstarlet sel -t -m //pgmap/num_pgs -v . ../qa/workunits/ceph-helpers.sh:952: is_clean: num_pgs=4 ../qa/workunits/ceph-helpers.sh:953: is_clean: test 4 '!=' 0 .../qa/workunits/ceph-helpers.sh:954: is_clean: get_num_active_clean .../qa/workunits/ceph-helpers.sh:863: get_num_active_clean: local 'expression=(' .../qa/workunits/ceph-helpers.sh:864: get_num_active_clean: expression+='contains(.,'\''active'\'') and ' .../qa/workunits/ceph-helpers.sh:865: get_num_active_clean: expression+='contains(.,'\''clean'\'') and ' .../qa/workunits/ceph-helpers.sh:866: get_num_active_clean: expression+='not(contains(.,'\''stale'\''))' .../qa/workunits/ceph-helpers.sh:867: get_num_active_clean: expression+=')' .../qa/workunits/ceph-helpers.sh:871: get_num_active_clean: ceph --format xml pg dump pgs .../qa/workunits/ceph-helpers.sh:873: get_num_active_clean: grep -v '^$' .../qa/workunits/ceph-helpers.sh:872: get_num_active_clean: xmlstarlet sel -t -m '//pg_stat/state[(contains(.,'\''active'\'') and contains(.,'\''clean'\'') and not(contains(.,'\''stale'\'')))]' -v . -n .../qa/workunits/ceph-helpers.sh:873: get_num_active_clean: wc -l ../qa/workunits/ceph-helpers.sh:954: is_clean: test 0 = 4 ../qa/workunits/ceph-helpers.sh:954: is_clean: return 1 ../qa/workunits/ceph-helpers.sh:986: wait_for_clean: get_is_making_recovery_progress .../qa/workunits/ceph-helpers.sh:839: get_is_making_recovery_progress: ceph --format xml status .../qa/workunits/ceph-helpers.sh:839: get_is_making_recovery_progress: xmlstarlet sel -t -m //pgmap/recovering_keys_per_sec -v . -o ' ' -t -m //pgmap/recovering_bytes_per_sec -v . -o ' ' -t -m //pgmap/recovering_objects_per_sec -v . ../qa/workunits/ceph-helpers.sh:839: get_is_making_recovery_progress: local progress= ../qa/workunits/ceph-helpers.sh:840: get_is_making_recovery_progress: test -n '' ../qa/workunits/ceph-helpers.sh:988: wait_for_clean: (( timer >= 120 )) .../qa/workunits/ceph-helpers.sh:993: wait_for_clean: get_num_active_clean .../qa/workunits/ceph-helpers.sh:863: get_num_active_clean: local 'expression=(' .../qa/workunits/ceph-helpers.sh:864: get_num_active_clean: expression+='contains(.,'\''active'\'') and ' .../qa/workunits/ceph-helpers.sh:865: get_num_active_clean: expression+='contains(.,'\''clean'\'') and ' .../qa/workunits/ceph-helpers.sh:866: get_num_active_clean: expression+='not(contains(.,'\''stale'\''))' .../qa/workunits/ceph-helpers.sh:867: get_num_active_clean: expression+=')' .../qa/workunits/ceph-helpers.sh:871: get_num_active_clean: ceph --format xml pg dump pgs .../qa/workunits/ceph-helpers.sh:872: get_num_active_clean: xmlstarlet sel -t -m '//pg_stat/state[(contains(.,'\''active'\'') and contains(.,'\''clean'\'') and not(contains(.,'\''stale'\'')))]' -v . -n .../qa/workunits/ceph-helpers.sh:873: get_num_active_clean: grep -v '^$' .../qa/workunits/ceph-helpers.sh:873: get_num_active_clean: wc -l ../qa/workunits/ceph-helpers.sh:993: wait_for_clean: cur_active_clean=0 ../qa/workunits/ceph-helpers.sh:994: wait_for_clean: test 0 '!=' 0 ../qa/workunits/ceph-helpers.sh:998: wait_for_clean: sleep 1 .../qa/workunits/ceph-helpers.sh:999: wait_for_clean: expr 6 + 1 ../qa/workunits/ceph-helpers.sh:999: wait_for_clean: timer=7 ../qa/workunits/ceph-helpers.sh:985: wait_for_clean: is_clean .../qa/workunits/ceph-helpers.sh:952: is_clean: get_num_pgs .../qa/workunits/ceph-helpers.sh:898: get_num_pgs: ceph --format xml status .../qa/workunits/ceph-helpers.sh:899: get_num_pgs: xmlstarlet sel -t -m //pgmap/num_pgs -v . ../qa/workunits/ceph-helpers.sh:952: is_clean: num_pgs=4 ../qa/workunits/ceph-helpers.sh:953: is_clean: test 4 '!=' 0 .../qa/workunits/ceph-helpers.sh:954: is_clean: get_num_active_clean .../qa/workunits/ceph-helpers.sh:863: get_num_active_clean: local 'expression=(' .../qa/workunits/ceph-helpers.sh:864: get_num_active_clean: expression+='contains(.,'\''active'\'') and ' .../qa/workunits/ceph-helpers.sh:865: get_num_active_clean: expression+='contains(.,'\''clean'\'') and ' .../qa/workunits/ceph-helpers.sh:866: get_num_active_clean: expression+='not(contains(.,'\''stale'\''))' .../qa/workunits/ceph-helpers.sh:867: get_num_active_clean: expression+=')' .../qa/workunits/ceph-helpers.sh:871: get_num_active_clean: ceph --format xml pg dump pgs .../qa/workunits/ceph-helpers.sh:872: get_num_active_clean: xmlstarlet sel -t -m '//pg_stat/state[(contains(.,'\''active'\'') and contains(.,'\''clean'\'') and not(contains(.,'\''stale'\'')))]' -v . -n .../qa/workunits/ceph-helpers.sh:873: get_num_active_clean: grep -v '^$' .../qa/workunits/ceph-helpers.sh:873: get_num_active_clean: wc -l ../qa/workunits/ceph-helpers.sh:954: is_clean: test 0 = 4 ../qa/workunits/ceph-helpers.sh:954: is_clean: return 1 ../qa/workunits/ceph-helpers.sh:986: wait_for_clean: get_is_making_recovery_progress .../qa/workunits/ceph-helpers.sh:839: get_is_making_recovery_progress: ceph --format xml status .../qa/workunits/ceph-helpers.sh:839: get_is_making_recovery_progress: xmlstarlet sel -t -m //pgmap/recovering_keys_per_sec -v . -o ' ' -t -m //pgmap/recovering_bytes_per_sec -v . -o ' ' -t -m //pgmap/recovering_objects_per_sec -v . ../qa/workunits/ceph-helpers.sh:839: get_is_making_recovery_progress: local progress= ../qa/workunits/ceph-helpers.sh:840: get_is_making_recovery_progress: test -n '' ../qa/workunits/ceph-helpers.sh:988: wait_for_clean: (( timer >= 120 )) .../qa/workunits/ceph-helpers.sh:993: wait_for_clean: get_num_active_clean .../qa/workunits/ceph-helpers.sh:863: get_num_active_clean: local 'expression=(' .../qa/workunits/ceph-helpers.sh:864: get_num_active_clean: expression+='contains(.,'\''active'\'') and ' .../qa/workunits/ceph-helpers.sh:865: get_num_active_clean: expression+='contains(.,'\''clean'\'') and ' .../qa/workunits/ceph-helpers.sh:866: get_num_active_clean: expression+='not(contains(.,'\''stale'\''))' .../qa/workunits/ceph-helpers.sh:867: get_num_active_clean: expression+=')' .../qa/workunits/ceph-helpers.sh:871: get_num_active_clean: ceph --format xml pg dump pgs .../qa/workunits/ceph-helpers.sh:872: get_num_active_clean: xmlstarlet sel -t -m '//pg_stat/state[(contains(.,'\''active'\'') and contains(.,'\''clean'\'') and not(contains(.,'\''stale'\'')))]' -v . -n .../qa/workunits/ceph-helpers.sh:873: get_num_active_clean: grep -v '^$' .../qa/workunits/ceph-helpers.sh:873: get_num_active_clean: wc -l ../qa/workunits/ceph-helpers.sh:993: wait_for_clean: cur_active_clean=0 ../qa/workunits/ceph-helpers.sh:994: wait_for_clean: test 0 '!=' 0 ../qa/workunits/ceph-helpers.sh:998: wait_for_clean: sleep 1 .../qa/workunits/ceph-helpers.sh:999: wait_for_clean: expr 7 + 1 ../qa/workunits/ceph-helpers.sh:999: wait_for_clean: timer=8 ../qa/workunits/ceph-helpers.sh:985: wait_for_clean: is_clean .../qa/workunits/ceph-helpers.sh:952: is_clean: get_num_pgs .../qa/workunits/ceph-helpers.sh:898: get_num_pgs: ceph --format xml status .../qa/workunits/ceph-helpers.sh:899: get_num_pgs: xmlstarlet sel -t -m //pgmap/num_pgs -v . ../qa/workunits/ceph-helpers.sh:952: is_clean: num_pgs=4 ../qa/workunits/ceph-helpers.sh:953: is_clean: test 4 '!=' 0 .../qa/workunits/ceph-helpers.sh:954: is_clean: get_num_active_clean .../qa/workunits/ceph-helpers.sh:863: get_num_active_clean: local 'expression=(' .../qa/workunits/ceph-helpers.sh:864: get_num_active_clean: expression+='contains(.,'\''active'\'') and ' .../qa/workunits/ceph-helpers.sh:865: get_num_active_clean: expression+='contains(.,'\''clean'\'') and ' .../qa/workunits/ceph-helpers.sh:866: get_num_active_clean: expression+='not(contains(.,'\''stale'\''))' .../qa/workunits/ceph-helpers.sh:867: get_num_active_clean: expression+=')' .../qa/workunits/ceph-helpers.sh:871: get_num_active_clean: ceph --format xml pg dump pgs .../qa/workunits/ceph-helpers.sh:872: get_num_active_clean: xmlstarlet sel -t -m '//pg_stat/state[(contains(.,'\''active'\'') and contains(.,'\''clean'\'') and not(contains(.,'\''stale'\'')))]' -v . -n .../qa/workunits/ceph-helpers.sh:873: get_num_active_clean: wc -l .../qa/workunits/ceph-helpers.sh:873: get_num_active_clean: grep -v '^$' ../qa/workunits/ceph-helpers.sh:954: is_clean: test 0 = 4 ../qa/workunits/ceph-helpers.sh:954: is_clean: return 1 ../qa/workunits/ceph-helpers.sh:986: wait_for_clean: get_is_making_recovery_progress .../qa/workunits/ceph-helpers.sh:839: get_is_making_recovery_progress: ceph --format xml status .../qa/workunits/ceph-helpers.sh:839: get_is_making_recovery_progress: xmlstarlet sel -t -m //pgmap/recovering_keys_per_sec -v . -o ' ' -t -m //pgmap/recovering_bytes_per_sec -v . -o ' ' -t -m //pgmap/recovering_objects_per_sec -v . ../qa/workunits/ceph-helpers.sh:839: get_is_making_recovery_progress: local progress= ../qa/workunits/ceph-helpers.sh:840: get_is_making_recovery_progress: test -n '' ../qa/workunits/ceph-helpers.sh:988: wait_for_clean: (( timer >= 120 )) .../qa/workunits/ceph-helpers.sh:993: wait_for_clean: get_num_active_clean .../qa/workunits/ceph-helpers.sh:863: get_num_active_clean: local 'expression=(' .../qa/workunits/ceph-helpers.sh:864: get_num_active_clean: expression+='contains(.,'\''active'\'') and ' .../qa/workunits/ceph-helpers.sh:865: get_num_active_clean: expression+='contains(.,'\''clean'\'') and ' .../qa/workunits/ceph-helpers.sh:866: get_num_active_clean: expression+='not(contains(.,'\''stale'\''))' .../qa/workunits/ceph-helpers.sh:867: get_num_active_clean: expression+=')' .../qa/workunits/ceph-helpers.sh:871: get_num_active_clean: ceph --format xml pg dump pgs .../qa/workunits/ceph-helpers.sh:872: get_num_active_clean: xmlstarlet sel -t -m '//pg_stat/state[(contains(.,'\''active'\'') and contains(.,'\''clean'\'') and not(contains(.,'\''stale'\'')))]' -v . -n .../qa/workunits/ceph-helpers.sh:873: get_num_active_clean: grep -v '^$' .../qa/workunits/ceph-helpers.sh:873: get_num_active_clean: wc -l ../qa/workunits/ceph-helpers.sh:993: wait_for_clean: cur_active_clean=0 ../qa/workunits/ceph-helpers.sh:994: wait_for_clean: test 0 '!=' 0 ../qa/workunits/ceph-helpers.sh:998: wait_for_clean: sleep 1 .../qa/workunits/ceph-helpers.sh:999: wait_for_clean: expr 8 + 1 ../qa/workunits/ceph-helpers.sh:999: wait_for_clean: timer=9 ../qa/workunits/ceph-helpers.sh:985: wait_for_clean: is_clean .../qa/workunits/ceph-helpers.sh:952: is_clean: get_num_pgs .../qa/workunits/ceph-helpers.sh:898: get_num_pgs: ceph --format xml status .../qa/workunits/ceph-helpers.sh:899: get_num_pgs: xmlstarlet sel -t -m //pgmap/num_pgs -v . ../qa/workunits/ceph-helpers.sh:952: is_clean: num_pgs=4 ../qa/workunits/ceph-helpers.sh:953: is_clean: test 4 '!=' 0 .../qa/workunits/ceph-helpers.sh:954: is_clean: get_num_active_clean .../qa/workunits/ceph-helpers.sh:863: get_num_active_clean: local 'expression=(' .../qa/workunits/ceph-helpers.sh:864: get_num_active_clean: expression+='contains(.,'\''active'\'') and ' .../qa/workunits/ceph-helpers.sh:865: get_num_active_clean: expression+='contains(.,'\''clean'\'') and ' .../qa/workunits/ceph-helpers.sh:866: get_num_active_clean: expression+='not(contains(.,'\''stale'\''))' .../qa/workunits/ceph-helpers.sh:867: get_num_active_clean: expression+=')' .../qa/workunits/ceph-helpers.sh:871: get_num_active_clean: ceph --format xml pg dump pgs .../qa/workunits/ceph-helpers.sh:872: get_num_active_clean: xmlstarlet sel -t -m '//pg_stat/state[(contains(.,'\''active'\'') and contains(.,'\''clean'\'') and not(contains(.,'\''stale'\'')))]' -v . -n .../qa/workunits/ceph-helpers.sh:873: get_num_active_clean: grep -v '^$' .../qa/workunits/ceph-helpers.sh:873: get_num_active_clean: wc -l ../qa/workunits/ceph-helpers.sh:954: is_clean: test 0 = 4 ../qa/workunits/ceph-helpers.sh:954: is_clean: return 1 ../qa/workunits/ceph-helpers.sh:986: wait_for_clean: get_is_making_recovery_progress .../qa/workunits/ceph-helpers.sh:839: get_is_making_recovery_progress: xmlstarlet sel -t -m //pgmap/recovering_keys_per_sec -v . -o ' ' -t -m //pgmap/recovering_bytes_per_sec -v . -o ' ' -t -m //pgmap/recovering_objects_per_sec -v . .../qa/workunits/ceph-helpers.sh:839: get_is_making_recovery_progress: ceph --format xml status ../qa/workunits/ceph-helpers.sh:839: get_is_making_recovery_progress: local progress= ../qa/workunits/ceph-helpers.sh:840: get_is_making_recovery_progress: test -n '' ../qa/workunits/ceph-helpers.sh:988: wait_for_clean: (( timer >= 120 )) .../qa/workunits/ceph-helpers.sh:993: wait_for_clean: get_num_active_clean .../qa/workunits/ceph-helpers.sh:863: get_num_active_clean: local 'expression=(' .../qa/workunits/ceph-helpers.sh:864: get_num_active_clean: expression+='contains(.,'\''active'\'') and ' .../qa/workunits/ceph-helpers.sh:865: get_num_active_clean: expression+='contains(.,'\''clean'\'') and ' .../qa/workunits/ceph-helpers.sh:866: get_num_active_clean: expression+='not(contains(.,'\''stale'\''))' .../qa/workunits/ceph-helpers.sh:867: get_num_active_clean: expression+=')' .../qa/workunits/ceph-helpers.sh:871: get_num_active_clean: ceph --format xml pg dump pgs .../qa/workunits/ceph-helpers.sh:872: get_num_active_clean: xmlstarlet sel -t -m '//pg_stat/state[(contains(.,'\''active'\'') and contains(.,'\''clean'\'') and not(contains(.,'\''stale'\'')))]' -v . -n .../qa/workunits/ceph-helpers.sh:873: get_num_active_clean: grep -v '^$' .../qa/workunits/ceph-helpers.sh:873: get_num_active_clean: wc -l ../qa/workunits/ceph-helpers.sh:993: wait_for_clean: cur_active_clean=0 ../qa/workunits/ceph-helpers.sh:994: wait_for_clean: test 0 '!=' 0 ../qa/workunits/ceph-helpers.sh:998: wait_for_clean: sleep 1 .../qa/workunits/ceph-helpers.sh:999: wait_for_clean: expr 9 + 1 ../qa/workunits/ceph-helpers.sh:999: wait_for_clean: timer=10 ../qa/workunits/ceph-helpers.sh:985: wait_for_clean: is_clean .../qa/workunits/ceph-helpers.sh:952: is_clean: get_num_pgs .../qa/workunits/ceph-helpers.sh:898: get_num_pgs: ceph --format xml status .../qa/workunits/ceph-helpers.sh:899: get_num_pgs: xmlstarlet sel -t -m //pgmap/num_pgs -v . ../qa/workunits/ceph-helpers.sh:952: is_clean: num_pgs=4 ../qa/workunits/ceph-helpers.sh:953: is_clean: test 4 '!=' 0 .../qa/workunits/ceph-helpers.sh:954: is_clean: get_num_active_clean .../qa/workunits/ceph-helpers.sh:863: get_num_active_clean: local 'expression=(' .../qa/workunits/ceph-helpers.sh:864: get_num_active_clean: expression+='contains(.,'\''active'\'') and ' .../qa/workunits/ceph-helpers.sh:865: get_num_active_clean: expression+='contains(.,'\''clean'\'') and ' .../qa/workunits/ceph-helpers.sh:866: get_num_active_clean: expression+='not(contains(.,'\''stale'\''))' .../qa/workunits/ceph-helpers.sh:867: get_num_active_clean: expression+=')' .../qa/workunits/ceph-helpers.sh:871: get_num_active_clean: ceph --format xml pg dump pgs .../qa/workunits/ceph-helpers.sh:872: get_num_active_clean: xmlstarlet sel -t -m '//pg_stat/state[(contains(.,'\''active'\'') and contains(.,'\''clean'\'') and not(contains(.,'\''stale'\'')))]' -v . -n .../qa/workunits/ceph-helpers.sh:873: get_num_active_clean: grep -v '^$' .../qa/workunits/ceph-helpers.sh:873: get_num_active_clean: wc -l ../qa/workunits/ceph-helpers.sh:954: is_clean: test 0 = 4 ../qa/workunits/ceph-helpers.sh:954: is_clean: return 1 ../qa/workunits/ceph-helpers.sh:986: wait_for_clean: get_is_making_recovery_progress .../qa/workunits/ceph-helpers.sh:839: get_is_making_recovery_progress: ceph --format xml status .../qa/workunits/ceph-helpers.sh:839: get_is_making_recovery_progress: xmlstarlet sel -t -m //pgmap/recovering_keys_per_sec -v . -o ' ' -t -m //pgmap/recovering_bytes_per_sec -v . -o ' ' -t -m //pgmap/recovering_objects_per_sec -v . ../qa/workunits/ceph-helpers.sh:839: get_is_making_recovery_progress: local progress= ../qa/workunits/ceph-helpers.sh:840: get_is_making_recovery_progress: test -n '' ../qa/workunits/ceph-helpers.sh:988: wait_for_clean: (( timer >= 120 )) .../qa/workunits/ceph-helpers.sh:993: wait_for_clean: get_num_active_clean .../qa/workunits/ceph-helpers.sh:863: get_num_active_clean: local 'expression=(' .../qa/workunits/ceph-helpers.sh:864: get_num_active_clean: expression+='contains(.,'\''active'\'') and ' .../qa/workunits/ceph-helpers.sh:865: get_num_active_clean: expression+='contains(.,'\''clean'\'') and ' .../qa/workunits/ceph-helpers.sh:866: get_num_active_clean: expression+='not(contains(.,'\''stale'\''))' .../qa/workunits/ceph-helpers.sh:867: get_num_active_clean: expression+=')' .../qa/workunits/ceph-helpers.sh:871: get_num_active_clean: ceph --format xml pg dump pgs .../qa/workunits/ceph-helpers.sh:872: get_num_active_clean: xmlstarlet sel -t -m '//pg_stat/state[(contains(.,'\''active'\'') and contains(.,'\''clean'\'') and not(contains(.,'\''stale'\'')))]' -v . -n .../qa/workunits/ceph-helpers.sh:873: get_num_active_clean: grep -v '^$' .../qa/workunits/ceph-helpers.sh:873: get_num_active_clean: wc -l ../qa/workunits/ceph-helpers.sh:993: wait_for_clean: cur_active_clean=0 ../qa/workunits/ceph-helpers.sh:994: wait_for_clean: test 0 '!=' 0 ../qa/workunits/ceph-helpers.sh:998: wait_for_clean: sleep 1 .../qa/workunits/ceph-helpers.sh:999: wait_for_clean: expr 10 + 1 ../qa/workunits/ceph-helpers.sh:999: wait_for_clean: timer=11 ../qa/workunits/ceph-helpers.sh:985: wait_for_clean: is_clean .../qa/workunits/ceph-helpers.sh:952: is_clean: get_num_pgs .../qa/workunits/ceph-helpers.sh:898: get_num_pgs: ceph --format xml status .../qa/workunits/ceph-helpers.sh:899: get_num_pgs: xmlstarlet sel -t -m //pgmap/num_pgs -v . ../qa/workunits/ceph-helpers.sh:952: is_clean: num_pgs=4 ../qa/workunits/ceph-helpers.sh:953: is_clean: test 4 '!=' 0 .../qa/workunits/ceph-helpers.sh:954: is_clean: get_num_active_clean .../qa/workunits/ceph-helpers.sh:863: get_num_active_clean: local 'expression=(' .../qa/workunits/ceph-helpers.sh:864: get_num_active_clean: expression+='contains(.,'\''active'\'') and ' .../qa/workunits/ceph-helpers.sh:865: get_num_active_clean: expression+='contains(.,'\''clean'\'') and ' .../qa/workunits/ceph-helpers.sh:866: get_num_active_clean: expression+='not(contains(.,'\''stale'\''))' .../qa/workunits/ceph-helpers.sh:867: get_num_active_clean: expression+=')' .../qa/workunits/ceph-helpers.sh:871: get_num_active_clean: ceph --format xml pg dump pgs .../qa/workunits/ceph-helpers.sh:872: get_num_active_clean: xmlstarlet sel -t -m '//pg_stat/state[(contains(.,'\''active'\'') and contains(.,'\''clean'\'') and not(contains(.,'\''stale'\'')))]' -v . -n .../qa/workunits/ceph-helpers.sh:873: get_num_active_clean: grep -v '^$' .../qa/workunits/ceph-helpers.sh:873: get_num_active_clean: wc -l ../qa/workunits/ceph-helpers.sh:954: is_clean: test 0 = 4 ../qa/workunits/ceph-helpers.sh:954: is_clean: return 1 ../qa/workunits/ceph-helpers.sh:986: wait_for_clean: get_is_making_recovery_progress .../qa/workunits/ceph-helpers.sh:839: get_is_making_recovery_progress: ceph --format xml status .../qa/workunits/ceph-helpers.sh:839: get_is_making_recovery_progress: xmlstarlet sel -t -m //pgmap/recovering_keys_per_sec -v . -o ' ' -t -m //pgmap/recovering_bytes_per_sec -v . -o ' ' -t -m //pgmap/recovering_objects_per_sec -v . ../qa/workunits/ceph-helpers.sh:839: get_is_making_recovery_progress: local progress= ../qa/workunits/ceph-helpers.sh:840: get_is_making_recovery_progress: test -n '' ../qa/workunits/ceph-helpers.sh:988: wait_for_clean: (( timer >= 120 )) .../qa/workunits/ceph-helpers.sh:993: wait_for_clean: get_num_active_clean .../qa/workunits/ceph-helpers.sh:863: get_num_active_clean: local 'expression=(' .../qa/workunits/ceph-helpers.sh:864: get_num_active_clean: expression+='contains(.,'\''active'\'') and ' .../qa/workunits/ceph-helpers.sh:865: get_num_active_clean: expression+='contains(.,'\''clean'\'') and ' .../qa/workunits/ceph-helpers.sh:866: get_num_active_clean: expression+='not(contains(.,'\''stale'\''))' .../qa/workunits/ceph-helpers.sh:867: get_num_active_clean: expression+=')' .../qa/workunits/ceph-helpers.sh:871: get_num_active_clean: ceph --format xml pg dump pgs .../qa/workunits/ceph-helpers.sh:872: get_num_active_clean: xmlstarlet sel -t -m '//pg_stat/state[(contains(.,'\''active'\'') and contains(.,'\''clean'\'') and not(contains(.,'\''stale'\'')))]' -v . -n .../qa/workunits/ceph-helpers.sh:873: get_num_active_clean: grep -v '^$' .../qa/workunits/ceph-helpers.sh:873: get_num_active_clean: wc -l ../qa/workunits/ceph-helpers.sh:993: wait_for_clean: cur_active_clean=0 ../qa/workunits/ceph-helpers.sh:994: wait_for_clean: test 0 '!=' 0 ../qa/workunits/ceph-helpers.sh:998: wait_for_clean: sleep 1 .../qa/workunits/ceph-helpers.sh:999: wait_for_clean: expr 11 + 1 ../qa/workunits/ceph-helpers.sh:999: wait_for_clean: timer=12 ../qa/workunits/ceph-helpers.sh:985: wait_for_clean: is_clean .../qa/workunits/ceph-helpers.sh:952: is_clean: get_num_pgs .../qa/workunits/ceph-helpers.sh:898: get_num_pgs: ceph --format xml status .../qa/workunits/ceph-helpers.sh:899: get_num_pgs: xmlstarlet sel -t -m //pgmap/num_pgs -v . ../qa/workunits/ceph-helpers.sh:952: is_clean: num_pgs=4 ../qa/workunits/ceph-helpers.sh:953: is_clean: test 4 '!=' 0 .../qa/workunits/ceph-helpers.sh:954: is_clean: get_num_active_clean .../qa/workunits/ceph-helpers.sh:863: get_num_active_clean: local 'expression=(' .../qa/workunits/ceph-helpers.sh:864: get_num_active_clean: expression+='contains(.,'\''active'\'') and ' .../qa/workunits/ceph-helpers.sh:865: get_num_active_clean: expression+='contains(.,'\''clean'\'') and ' .../qa/workunits/ceph-helpers.sh:866: get_num_active_clean: expression+='not(contains(.,'\''stale'\''))' .../qa/workunits/ceph-helpers.sh:867: get_num_active_clean: expression+=')' .../qa/workunits/ceph-helpers.sh:871: get_num_active_clean: ceph --format xml pg dump pgs .../qa/workunits/ceph-helpers.sh:872: get_num_active_clean: xmlstarlet sel -t -m '//pg_stat/state[(contains(.,'\''active'\'') and contains(.,'\''clean'\'') and not(contains(.,'\''stale'\'')))]' -v . -n .../qa/workunits/ceph-helpers.sh:873: get_num_active_clean: grep -v '^$' .../qa/workunits/ceph-helpers.sh:873: get_num_active_clean: wc -l ../qa/workunits/ceph-helpers.sh:954: is_clean: test 0 = 4 ../qa/workunits/ceph-helpers.sh:954: is_clean: return 1 ../qa/workunits/ceph-helpers.sh:986: wait_for_clean: get_is_making_recovery_progress .../qa/workunits/ceph-helpers.sh:839: get_is_making_recovery_progress: ceph --format xml status .../qa/workunits/ceph-helpers.sh:839: get_is_making_recovery_progress: xmlstarlet sel -t -m //pgmap/recovering_keys_per_sec -v . -o ' ' -t -m //pgmap/recovering_bytes_per_sec -v . -o ' ' -t -m //pgmap/recovering_objects_per_sec -v . ../qa/workunits/ceph-helpers.sh:839: get_is_making_recovery_progress: local progress= ../qa/workunits/ceph-helpers.sh:840: get_is_making_recovery_progress: test -n '' ../qa/workunits/ceph-helpers.sh:988: wait_for_clean: (( timer >= 120 )) .../qa/workunits/ceph-helpers.sh:993: wait_for_clean: get_num_active_clean .../qa/workunits/ceph-helpers.sh:863: get_num_active_clean: local 'expression=(' .../qa/workunits/ceph-helpers.sh:864: get_num_active_clean: expression+='contains(.,'\''active'\'') and ' .../qa/workunits/ceph-helpers.sh:865: get_num_active_clean: expression+='contains(.,'\''clean'\'') and ' .../qa/workunits/ceph-helpers.sh:866: get_num_active_clean: expression+='not(contains(.,'\''stale'\''))' .../qa/workunits/ceph-helpers.sh:867: get_num_active_clean: expression+=')' .../qa/workunits/ceph-helpers.sh:871: get_num_active_clean: ceph --format xml pg dump pgs .../qa/workunits/ceph-helpers.sh:872: get_num_active_clean: xmlstarlet sel -t -m '//pg_stat/state[(contains(.,'\''active'\'') and contains(.,'\''clean'\'') and not(contains(.,'\''stale'\'')))]' -v . -n .../qa/workunits/ceph-helpers.sh:873: get_num_active_clean: grep -v '^$' .../qa/workunits/ceph-helpers.sh:873: get_num_active_clean: wc -l ../qa/workunits/ceph-helpers.sh:993: wait_for_clean: cur_active_clean=0 ../qa/workunits/ceph-helpers.sh:994: wait_for_clean: test 0 '!=' 0 ../qa/workunits/ceph-helpers.sh:998: wait_for_clean: sleep 1 .../qa/workunits/ceph-helpers.sh:999: wait_for_clean: expr 12 + 1 ../qa/workunits/ceph-helpers.sh:999: wait_for_clean: timer=13 ../qa/workunits/ceph-helpers.sh:985: wait_for_clean: is_clean .../qa/workunits/ceph-helpers.sh:952: is_clean: get_num_pgs .../qa/workunits/ceph-helpers.sh:898: get_num_pgs: ceph --format xml status .../qa/workunits/ceph-helpers.sh:899: get_num_pgs: xmlstarlet sel -t -m //pgmap/num_pgs -v . ../qa/workunits/ceph-helpers.sh:952: is_clean: num_pgs=4 ../qa/workunits/ceph-helpers.sh:953: is_clean: test 4 '!=' 0 .../qa/workunits/ceph-helpers.sh:954: is_clean: get_num_active_clean .../qa/workunits/ceph-helpers.sh:863: get_num_active_clean: local 'expression=(' .../qa/workunits/ceph-helpers.sh:864: get_num_active_clean: expression+='contains(.,'\''active'\'') and ' .../qa/workunits/ceph-helpers.sh:865: get_num_active_clean: expression+='contains(.,'\''clean'\'') and ' .../qa/workunits/ceph-helpers.sh:866: get_num_active_clean: expression+='not(contains(.,'\''stale'\''))' .../qa/workunits/ceph-helpers.sh:867: get_num_active_clean: expression+=')' .../qa/workunits/ceph-helpers.sh:871: get_num_active_clean: ceph --format xml pg dump pgs .../qa/workunits/ceph-helpers.sh:872: get_num_active_clean: xmlstarlet sel -t -m '//pg_stat/state[(contains(.,'\''active'\'') and contains(.,'\''clean'\'') and not(contains(.,'\''stale'\'')))]' -v . -n .../qa/workunits/ceph-helpers.sh:873: get_num_active_clean: grep -v '^$' .../qa/workunits/ceph-helpers.sh:873: get_num_active_clean: wc -l ../qa/workunits/ceph-helpers.sh:954: is_clean: test 0 = 4 ../qa/workunits/ceph-helpers.sh:954: is_clean: return 1 ../qa/workunits/ceph-helpers.sh:986: wait_for_clean: get_is_making_recovery_progress .../qa/workunits/ceph-helpers.sh:839: get_is_making_recovery_progress: ceph --format xml status .../qa/workunits/ceph-helpers.sh:839: get_is_making_recovery_progress: xmlstarlet sel -t -m //pgmap/recovering_keys_per_sec -v . -o ' ' -t -m //pgmap/recovering_bytes_per_sec -v . -o ' ' -t -m //pgmap/recovering_objects_per_sec -v . ../qa/workunits/ceph-helpers.sh:839: get_is_making_recovery_progress: local progress= ../qa/workunits/ceph-helpers.sh:840: get_is_making_recovery_progress: test -n '' ../qa/workunits/ceph-helpers.sh:988: wait_for_clean: (( timer >= 120 )) .../qa/workunits/ceph-helpers.sh:993: wait_for_clean: get_num_active_clean .../qa/workunits/ceph-helpers.sh:863: get_num_active_clean: local 'expression=(' .../qa/workunits/ceph-helpers.sh:864: get_num_active_clean: expression+='contains(.,'\''active'\'') and ' .../qa/workunits/ceph-helpers.sh:865: get_num_active_clean: expression+='contains(.,'\''clean'\'') and ' .../qa/workunits/ceph-helpers.sh:866: get_num_active_clean: expression+='not(contains(.,'\''stale'\''))' .../qa/workunits/ceph-helpers.sh:867: get_num_active_clean: expression+=')' .../qa/workunits/ceph-helpers.sh:871: get_num_active_clean: ceph --format xml pg dump pgs .../qa/workunits/ceph-helpers.sh:872: get_num_active_clean: xmlstarlet sel -t -m '//pg_stat/state[(contains(.,'\''active'\'') and contains(.,'\''clean'\'') and not(contains(.,'\''stale'\'')))]' -v . -n .../qa/workunits/ceph-helpers.sh:873: get_num_active_clean: wc -l .../qa/workunits/ceph-helpers.sh:873: get_num_active_clean: grep -v '^$' ../qa/workunits/ceph-helpers.sh:993: wait_for_clean: cur_active_clean=0 ../qa/workunits/ceph-helpers.sh:994: wait_for_clean: test 0 '!=' 0 ../qa/workunits/ceph-helpers.sh:998: wait_for_clean: sleep 1 .../qa/workunits/ceph-helpers.sh:999: wait_for_clean: expr 13 + 1 ../qa/workunits/ceph-helpers.sh:999: wait_for_clean: timer=14 ../qa/workunits/ceph-helpers.sh:985: wait_for_clean: is_clean .../qa/workunits/ceph-helpers.sh:952: is_clean: get_num_pgs .../qa/workunits/ceph-helpers.sh:898: get_num_pgs: ceph --format xml status .../qa/workunits/ceph-helpers.sh:899: get_num_pgs: xmlstarlet sel -t -m //pgmap/num_pgs -v . ../qa/workunits/ceph-helpers.sh:952: is_clean: num_pgs=4 ../qa/workunits/ceph-helpers.sh:953: is_clean: test 4 '!=' 0 .../qa/workunits/ceph-helpers.sh:954: is_clean: get_num_active_clean .../qa/workunits/ceph-helpers.sh:863: get_num_active_clean: local 'expression=(' .../qa/workunits/ceph-helpers.sh:864: get_num_active_clean: expression+='contains(.,'\''active'\'') and ' .../qa/workunits/ceph-helpers.sh:865: get_num_active_clean: expression+='contains(.,'\''clean'\'') and ' .../qa/workunits/ceph-helpers.sh:866: get_num_active_clean: expression+='not(contains(.,'\''stale'\''))' .../qa/workunits/ceph-helpers.sh:867: get_num_active_clean: expression+=')' .../qa/workunits/ceph-helpers.sh:871: get_num_active_clean: ceph --format xml pg dump pgs .../qa/workunits/ceph-helpers.sh:872: get_num_active_clean: xmlstarlet sel -t -m '//pg_stat/state[(contains(.,'\''active'\'') and contains(.,'\''clean'\'') and not(contains(.,'\''stale'\'')))]' -v . -n .../qa/workunits/ceph-helpers.sh:873: get_num_active_clean: grep -v '^$' .../qa/workunits/ceph-helpers.sh:873: get_num_active_clean: wc -l ../qa/workunits/ceph-helpers.sh:954: is_clean: test 0 = 4 ../qa/workunits/ceph-helpers.sh:954: is_clean: return 1 ../qa/workunits/ceph-helpers.sh:986: wait_for_clean: get_is_making_recovery_progress .../qa/workunits/ceph-helpers.sh:839: get_is_making_recovery_progress: ceph --format xml status .../qa/workunits/ceph-helpers.sh:839: get_is_making_recovery_progress: xmlstarlet sel -t -m //pgmap/recovering_keys_per_sec -v . -o ' ' -t -m //pgmap/recovering_bytes_per_sec -v . -o ' ' -t -m //pgmap/recovering_objects_per_sec -v . ../qa/workunits/ceph-helpers.sh:839: get_is_making_recovery_progress: local progress= ../qa/workunits/ceph-helpers.sh:840: get_is_making_recovery_progress: test -n '' ../qa/workunits/ceph-helpers.sh:988: wait_for_clean: (( timer >= 120 )) .../qa/workunits/ceph-helpers.sh:993: wait_for_clean: get_num_active_clean .../qa/workunits/ceph-helpers.sh:863: get_num_active_clean: local 'expression=(' .../qa/workunits/ceph-helpers.sh:864: get_num_active_clean: expression+='contains(.,'\''active'\'') and ' .../qa/workunits/ceph-helpers.sh:865: get_num_active_clean: expression+='contains(.,'\''clean'\'') and ' .../qa/workunits/ceph-helpers.sh:866: get_num_active_clean: expression+='not(contains(.,'\''stale'\''))' .../qa/workunits/ceph-helpers.sh:867: get_num_active_clean: expression+=')' .../qa/workunits/ceph-helpers.sh:871: get_num_active_clean: ceph --format xml pg dump pgs .../qa/workunits/ceph-helpers.sh:873: get_num_active_clean: wc -l .../qa/workunits/ceph-helpers.sh:872: get_num_active_clean: xmlstarlet sel -t -m '//pg_stat/state[(contains(.,'\''active'\'') and contains(.,'\''clean'\'') and not(contains(.,'\''stale'\'')))]' -v . -n .../qa/workunits/ceph-helpers.sh:873: get_num_active_clean: grep -v '^$' ../qa/workunits/ceph-helpers.sh:993: wait_for_clean: cur_active_clean=0 ../qa/workunits/ceph-helpers.sh:994: wait_for_clean: test 0 '!=' 0 ../qa/workunits/ceph-helpers.sh:998: wait_for_clean: sleep 1 .../qa/workunits/ceph-helpers.sh:999: wait_for_clean: expr 14 + 1 ../qa/workunits/ceph-helpers.sh:999: wait_for_clean: timer=15 ../qa/workunits/ceph-helpers.sh:985: wait_for_clean: is_clean .../qa/workunits/ceph-helpers.sh:952: is_clean: get_num_pgs .../qa/workunits/ceph-helpers.sh:898: get_num_pgs: ceph --format xml status .../qa/workunits/ceph-helpers.sh:899: get_num_pgs: xmlstarlet sel -t -m //pgmap/num_pgs -v . ../qa/workunits/ceph-helpers.sh:952: is_clean: num_pgs=4 ../qa/workunits/ceph-helpers.sh:953: is_clean: test 4 '!=' 0 .../qa/workunits/ceph-helpers.sh:954: is_clean: get_num_active_clean .../qa/workunits/ceph-helpers.sh:863: get_num_active_clean: local 'expression=(' .../qa/workunits/ceph-helpers.sh:864: get_num_active_clean: expression+='contains(.,'\''active'\'') and ' .../qa/workunits/ceph-helpers.sh:865: get_num_active_clean: expression+='contains(.,'\''clean'\'') and ' .../qa/workunits/ceph-helpers.sh:866: get_num_active_clean: expression+='not(contains(.,'\''stale'\''))' .../qa/workunits/ceph-helpers.sh:867: get_num_active_clean: expression+=')' .../qa/workunits/ceph-helpers.sh:871: get_num_active_clean: ceph --format xml pg dump pgs .../qa/workunits/ceph-helpers.sh:872: get_num_active_clean: xmlstarlet sel -t -m '//pg_stat/state[(contains(.,'\''active'\'') and contains(.,'\''clean'\'') and not(contains(.,'\''stale'\'')))]' -v . -n .../qa/workunits/ceph-helpers.sh:873: get_num_active_clean: grep -v '^$' .../qa/workunits/ceph-helpers.sh:873: get_num_active_clean: wc -l ../qa/workunits/ceph-helpers.sh:954: is_clean: test 0 = 4 ../qa/workunits/ceph-helpers.sh:954: is_clean: return 1 ../qa/workunits/ceph-helpers.sh:986: wait_for_clean: get_is_making_recovery_progress .../qa/workunits/ceph-helpers.sh:839: get_is_making_recovery_progress: ceph --format xml status .../qa/workunits/ceph-helpers.sh:839: get_is_making_recovery_progress: xmlstarlet sel -t -m //pgmap/recovering_keys_per_sec -v . -o ' ' -t -m //pgmap/recovering_bytes_per_sec -v . -o ' ' -t -m //pgmap/recovering_objects_per_sec -v . ../qa/workunits/ceph-helpers.sh:839: get_is_making_recovery_progress: local progress= ../qa/workunits/ceph-helpers.sh:840: get_is_making_recovery_progress: test -n '' ../qa/workunits/ceph-helpers.sh:988: wait_for_clean: (( timer >= 120 )) .../qa/workunits/ceph-helpers.sh:993: wait_for_clean: get_num_active_clean .../qa/workunits/ceph-helpers.sh:863: get_num_active_clean: local 'expression=(' .../qa/workunits/ceph-helpers.sh:864: get_num_active_clean: expression+='contains(.,'\''active'\'') and ' .../qa/workunits/ceph-helpers.sh:865: get_num_active_clean: expression+='contains(.,'\''clean'\'') and ' .../qa/workunits/ceph-helpers.sh:866: get_num_active_clean: expression+='not(contains(.,'\''stale'\''))' .../qa/workunits/ceph-helpers.sh:867: get_num_active_clean: expression+=')' .../qa/workunits/ceph-helpers.sh:871: get_num_active_clean: ceph --format xml pg dump pgs .../qa/workunits/ceph-helpers.sh:872: get_num_active_clean: xmlstarlet sel -t -m '//pg_stat/state[(contains(.,'\''active'\'') and contains(.,'\''clean'\'') and not(contains(.,'\''stale'\'')))]' -v . -n .../qa/workunits/ceph-helpers.sh:873: get_num_active_clean: grep -v '^$' .../qa/workunits/ceph-helpers.sh:873: get_num_active_clean: wc -l ../qa/workunits/ceph-helpers.sh:993: wait_for_clean: cur_active_clean=0 ../qa/workunits/ceph-helpers.sh:994: wait_for_clean: test 0 '!=' 0 ../qa/workunits/ceph-helpers.sh:998: wait_for_clean: sleep 1 .../qa/workunits/ceph-helpers.sh:999: wait_for_clean: expr 15 + 1 ../qa/workunits/ceph-helpers.sh:999: wait_for_clean: timer=16 ../qa/workunits/ceph-helpers.sh:985: wait_for_clean: is_clean .../qa/workunits/ceph-helpers.sh:952: is_clean: get_num_pgs .../qa/workunits/ceph-helpers.sh:898: get_num_pgs: ceph --format xml status .../qa/workunits/ceph-helpers.sh:899: get_num_pgs: xmlstarlet sel -t -m //pgmap/num_pgs -v . ../qa/workunits/ceph-helpers.sh:952: is_clean: num_pgs=4 ../qa/workunits/ceph-helpers.sh:953: is_clean: test 4 '!=' 0 .../qa/workunits/ceph-helpers.sh:954: is_clean: get_num_active_clean .../qa/workunits/ceph-helpers.sh:863: get_num_active_clean: local 'expression=(' .../qa/workunits/ceph-helpers.sh:864: get_num_active_clean: expression+='contains(.,'\''active'\'') and ' .../qa/workunits/ceph-helpers.sh:865: get_num_active_clean: expression+='contains(.,'\''clean'\'') and ' .../qa/workunits/ceph-helpers.sh:866: get_num_active_clean: expression+='not(contains(.,'\''stale'\''))' .../qa/workunits/ceph-helpers.sh:867: get_num_active_clean: expression+=')' .../qa/workunits/ceph-helpers.sh:871: get_num_active_clean: ceph --format xml pg dump pgs .../qa/workunits/ceph-helpers.sh:872: get_num_active_clean: xmlstarlet sel -t -m '//pg_stat/state[(contains(.,'\''active'\'') and contains(.,'\''clean'\'') and not(contains(.,'\''stale'\'')))]' -v . -n .../qa/workunits/ceph-helpers.sh:873: get_num_active_clean: grep -v '^$' .../qa/workunits/ceph-helpers.sh:873: get_num_active_clean: wc -l ../qa/workunits/ceph-helpers.sh:954: is_clean: test 0 = 4 ../qa/workunits/ceph-helpers.sh:954: is_clean: return 1 ../qa/workunits/ceph-helpers.sh:986: wait_for_clean: get_is_making_recovery_progress .../qa/workunits/ceph-helpers.sh:839: get_is_making_recovery_progress: ceph --format xml status .../qa/workunits/ceph-helpers.sh:839: get_is_making_recovery_progress: xmlstarlet sel -t -m //pgmap/recovering_keys_per_sec -v . -o ' ' -t -m //pgmap/recovering_bytes_per_sec -v . -o ' ' -t -m //pgmap/recovering_objects_per_sec -v . ../qa/workunits/ceph-helpers.sh:839: get_is_making_recovery_progress: local progress= ../qa/workunits/ceph-helpers.sh:840: get_is_making_recovery_progress: test -n '' ../qa/workunits/ceph-helpers.sh:988: wait_for_clean: (( timer >= 120 )) .../qa/workunits/ceph-helpers.sh:993: wait_for_clean: get_num_active_clean .../qa/workunits/ceph-helpers.sh:863: get_num_active_clean: local 'expression=(' .../qa/workunits/ceph-helpers.sh:864: get_num_active_clean: expression+='contains(.,'\''active'\'') and ' .../qa/workunits/ceph-helpers.sh:865: get_num_active_clean: expression+='contains(.,'\''clean'\'') and ' .../qa/workunits/ceph-helpers.sh:866: get_num_active_clean: expression+='not(contains(.,'\''stale'\''))' .../qa/workunits/ceph-helpers.sh:867: get_num_active_clean: expression+=')' .../qa/workunits/ceph-helpers.sh:871: get_num_active_clean: ceph --format xml pg dump pgs .../qa/workunits/ceph-helpers.sh:873: get_num_active_clean: grep -v '^$' .../qa/workunits/ceph-helpers.sh:872: get_num_active_clean: xmlstarlet sel -t -m '//pg_stat/state[(contains(.,'\''active'\'') and contains(.,'\''clean'\'') and not(contains(.,'\''stale'\'')))]' -v . -n .../qa/workunits/ceph-helpers.sh:873: get_num_active_clean: wc -l ../qa/workunits/ceph-helpers.sh:993: wait_for_clean: cur_active_clean=0 ../qa/workunits/ceph-helpers.sh:994: wait_for_clean: test 0 '!=' 0 ../qa/workunits/ceph-helpers.sh:998: wait_for_clean: sleep 1 .../qa/workunits/ceph-helpers.sh:999: wait_for_clean: expr 16 + 1 ../qa/workunits/ceph-helpers.sh:999: wait_for_clean: timer=17 ../qa/workunits/ceph-helpers.sh:985: wait_for_clean: is_clean .../qa/workunits/ceph-helpers.sh:952: is_clean: get_num_pgs .../qa/workunits/ceph-helpers.sh:898: get_num_pgs: ceph --format xml status .../qa/workunits/ceph-helpers.sh:899: get_num_pgs: xmlstarlet sel -t -m //pgmap/num_pgs -v . ../qa/workunits/ceph-helpers.sh:952: is_clean: num_pgs=4 ../qa/workunits/ceph-helpers.sh:953: is_clean: test 4 '!=' 0 .../qa/workunits/ceph-helpers.sh:954: is_clean: get_num_active_clean .../qa/workunits/ceph-helpers.sh:863: get_num_active_clean: local 'expression=(' .../qa/workunits/ceph-helpers.sh:864: get_num_active_clean: expression+='contains(.,'\''active'\'') and ' .../qa/workunits/ceph-helpers.sh:865: get_num_active_clean: expression+='contains(.,'\''clean'\'') and ' .../qa/workunits/ceph-helpers.sh:866: get_num_active_clean: expression+='not(contains(.,'\''stale'\''))' .../qa/workunits/ceph-helpers.sh:867: get_num_active_clean: expression+=')' .../qa/workunits/ceph-helpers.sh:871: get_num_active_clean: ceph --format xml pg dump pgs .../qa/workunits/ceph-helpers.sh:872: get_num_active_clean: xmlstarlet sel -t -m '//pg_stat/state[(contains(.,'\''active'\'') and contains(.,'\''clean'\'') and not(contains(.,'\''stale'\'')))]' -v . -n .../qa/workunits/ceph-helpers.sh:873: get_num_active_clean: grep -v '^$' .../qa/workunits/ceph-helpers.sh:873: get_num_active_clean: wc -l ../qa/workunits/ceph-helpers.sh:954: is_clean: test 0 = 4 ../qa/workunits/ceph-helpers.sh:954: is_clean: return 1 ../qa/workunits/ceph-helpers.sh:986: wait_for_clean: get_is_making_recovery_progress .../qa/workunits/ceph-helpers.sh:839: get_is_making_recovery_progress: xmlstarlet sel -t -m //pgmap/recovering_keys_per_sec -v . -o ' ' -t -m //pgmap/recovering_bytes_per_sec -v . -o ' ' -t -m //pgmap/recovering_objects_per_sec -v . .../qa/workunits/ceph-helpers.sh:839: get_is_making_recovery_progress: ceph --format xml status ../qa/workunits/ceph-helpers.sh:839: get_is_making_recovery_progress: local progress= ../qa/workunits/ceph-helpers.sh:840: get_is_making_recovery_progress: test -n '' ../qa/workunits/ceph-helpers.sh:988: wait_for_clean: (( timer >= 120 )) .../qa/workunits/ceph-helpers.sh:993: wait_for_clean: get_num_active_clean .../qa/workunits/ceph-helpers.sh:863: get_num_active_clean: local 'expression=(' .../qa/workunits/ceph-helpers.sh:864: get_num_active_clean: expression+='contains(.,'\''active'\'') and ' .../qa/workunits/ceph-helpers.sh:865: get_num_active_clean: expression+='contains(.,'\''clean'\'') and ' .../qa/workunits/ceph-helpers.sh:866: get_num_active_clean: expression+='not(contains(.,'\''stale'\''))' .../qa/workunits/ceph-helpers.sh:867: get_num_active_clean: expression+=')' .../qa/workunits/ceph-helpers.sh:873: get_num_active_clean: grep -v '^$' .../qa/workunits/ceph-helpers.sh:871: get_num_active_clean: ceph --format xml pg dump pgs .../qa/workunits/ceph-helpers.sh:873: get_num_active_clean: wc -l .../qa/workunits/ceph-helpers.sh:872: get_num_active_clean: xmlstarlet sel -t -m '//pg_stat/state[(contains(.,'\''active'\'') and contains(.,'\''clean'\'') and not(contains(.,'\''stale'\'')))]' -v . -n ../qa/workunits/ceph-helpers.sh:993: wait_for_clean: cur_active_clean=0 ../qa/workunits/ceph-helpers.sh:994: wait_for_clean: test 0 '!=' 0 ../qa/workunits/ceph-helpers.sh:998: wait_for_clean: sleep 1 .../qa/workunits/ceph-helpers.sh:999: wait_for_clean: expr 17 + 1 ../qa/workunits/ceph-helpers.sh:999: wait_for_clean: timer=18 ../qa/workunits/ceph-helpers.sh:985: wait_for_clean: is_clean .../qa/workunits/ceph-helpers.sh:952: is_clean: get_num_pgs .../qa/workunits/ceph-helpers.sh:899: get_num_pgs: xmlstarlet sel -t -m //pgmap/num_pgs -v . .../qa/workunits/ceph-helpers.sh:898: get_num_pgs: ceph --format xml status ../qa/workunits/ceph-helpers.sh:952: is_clean: num_pgs=4 ../qa/workunits/ceph-helpers.sh:953: is_clean: test 4 '!=' 0 .../qa/workunits/ceph-helpers.sh:954: is_clean: get_num_active_clean .../qa/workunits/ceph-helpers.sh:863: get_num_active_clean: local 'expression=(' .../qa/workunits/ceph-helpers.sh:864: get_num_active_clean: expression+='contains(.,'\''active'\'') and ' .../qa/workunits/ceph-helpers.sh:865: get_num_active_clean: expression+='contains(.,'\''clean'\'') and ' .../qa/workunits/ceph-helpers.sh:866: get_num_active_clean: expression+='not(contains(.,'\''stale'\''))' .../qa/workunits/ceph-helpers.sh:867: get_num_active_clean: expression+=')' .../qa/workunits/ceph-helpers.sh:871: get_num_active_clean: ceph --format xml pg dump pgs .../qa/workunits/ceph-helpers.sh:872: get_num_active_clean: xmlstarlet sel -t -m '//pg_stat/state[(contains(.,'\''active'\'') and contains(.,'\''clean'\'') and not(contains(.,'\''stale'\'')))]' -v . -n .../qa/workunits/ceph-helpers.sh:873: get_num_active_clean: grep -v '^$' .../qa/workunits/ceph-helpers.sh:873: get_num_active_clean: wc -l ../qa/workunits/ceph-helpers.sh:954: is_clean: test 0 = 4 ../qa/workunits/ceph-helpers.sh:954: is_clean: return 1 ../qa/workunits/ceph-helpers.sh:986: wait_for_clean: get_is_making_recovery_progress .../qa/workunits/ceph-helpers.sh:839: get_is_making_recovery_progress: ceph --format xml status .../qa/workunits/ceph-helpers.sh:839: get_is_making_recovery_progress: xmlstarlet sel -t -m //pgmap/recovering_keys_per_sec -v . -o ' ' -t -m //pgmap/recovering_bytes_per_sec -v . -o ' ' -t -m //pgmap/recovering_objects_per_sec -v . ../qa/workunits/ceph-helpers.sh:839: get_is_making_recovery_progress: local progress= ../qa/workunits/ceph-helpers.sh:840: get_is_making_recovery_progress: test -n '' ../qa/workunits/ceph-helpers.sh:988: wait_for_clean: (( timer >= 120 )) .../qa/workunits/ceph-helpers.sh:993: wait_for_clean: get_num_active_clean .../qa/workunits/ceph-helpers.sh:863: get_num_active_clean: local 'expression=(' .../qa/workunits/ceph-helpers.sh:864: get_num_active_clean: expression+='contains(.,'\''active'\'') and ' .../qa/workunits/ceph-helpers.sh:865: get_num_active_clean: expression+='contains(.,'\''clean'\'') and ' .../qa/workunits/ceph-helpers.sh:866: get_num_active_clean: expression+='not(contains(.,'\''stale'\''))' .../qa/workunits/ceph-helpers.sh:867: get_num_active_clean: expression+=')' .../qa/workunits/ceph-helpers.sh:871: get_num_active_clean: ceph --format xml pg dump pgs .../qa/workunits/ceph-helpers.sh:872: get_num_active_clean: xmlstarlet sel -t -m '//pg_stat/state[(contains(.,'\''active'\'') and contains(.,'\''clean'\'') and not(contains(.,'\''stale'\'')))]' -v . -n .../qa/workunits/ceph-helpers.sh:873: get_num_active_clean: wc -l .../qa/workunits/ceph-helpers.sh:873: get_num_active_clean: grep -v '^$' ../qa/workunits/ceph-helpers.sh:993: wait_for_clean: cur_active_clean=0 ../qa/workunits/ceph-helpers.sh:994: wait_for_clean: test 0 '!=' 0 ../qa/workunits/ceph-helpers.sh:998: wait_for_clean: sleep 1 .../qa/workunits/ceph-helpers.sh:999: wait_for_clean: expr 18 + 1 ../qa/workunits/ceph-helpers.sh:999: wait_for_clean: timer=19 ../qa/workunits/ceph-helpers.sh:985: wait_for_clean: is_clean .../qa/workunits/ceph-helpers.sh:952: is_clean: get_num_pgs .../qa/workunits/ceph-helpers.sh:898: get_num_pgs: ceph --format xml status .../qa/workunits/ceph-helpers.sh:899: get_num_pgs: xmlstarlet sel -t -m //pgmap/num_pgs -v . ../qa/workunits/ceph-helpers.sh:952: is_clean: num_pgs=4 ../qa/workunits/ceph-helpers.sh:953: is_clean: test 4 '!=' 0 .../qa/workunits/ceph-helpers.sh:954: is_clean: get_num_active_clean .../qa/workunits/ceph-helpers.sh:863: get_num_active_clean: local 'expression=(' .../qa/workunits/ceph-helpers.sh:864: get_num_active_clean: expression+='contains(.,'\''active'\'') and ' .../qa/workunits/ceph-helpers.sh:865: get_num_active_clean: expression+='contains(.,'\''clean'\'') and ' .../qa/workunits/ceph-helpers.sh:866: get_num_active_clean: expression+='not(contains(.,'\''stale'\''))' .../qa/workunits/ceph-helpers.sh:867: get_num_active_clean: expression+=')' .../qa/workunits/ceph-helpers.sh:871: get_num_active_clean: ceph --format xml pg dump pgs .../qa/workunits/ceph-helpers.sh:872: get_num_active_clean: xmlstarlet sel -t -m '//pg_stat/state[(contains(.,'\''active'\'') and contains(.,'\''clean'\'') and not(contains(.,'\''stale'\'')))]' -v . -n .../qa/workunits/ceph-helpers.sh:873: get_num_active_clean: grep -v '^$' .../qa/workunits/ceph-helpers.sh:873: get_num_active_clean: wc -l ../qa/workunits/ceph-helpers.sh:954: is_clean: test 0 = 4 ../qa/workunits/ceph-helpers.sh:954: is_clean: return 1 ../qa/workunits/ceph-helpers.sh:986: wait_for_clean: get_is_making_recovery_progress .../qa/workunits/ceph-helpers.sh:839: get_is_making_recovery_progress: ceph --format xml status .../qa/workunits/ceph-helpers.sh:839: get_is_making_recovery_progress: xmlstarlet sel -t -m //pgmap/recovering_keys_per_sec -v . -o ' ' -t -m //pgmap/recovering_bytes_per_sec -v . -o ' ' -t -m //pgmap/recovering_objects_per_sec -v . ../qa/workunits/ceph-helpers.sh:839: get_is_making_recovery_progress: local progress= ../qa/workunits/ceph-helpers.sh:840: get_is_making_recovery_progress: test -n '' ../qa/workunits/ceph-helpers.sh:988: wait_for_clean: (( timer >= 120 )) .../qa/workunits/ceph-helpers.sh:993: wait_for_clean: get_num_active_clean .../qa/workunits/ceph-helpers.sh:863: get_num_active_clean: local 'expression=(' .../qa/workunits/ceph-helpers.sh:864: get_num_active_clean: expression+='contains(.,'\''active'\'') and ' .../qa/workunits/ceph-helpers.sh:865: get_num_active_clean: expression+='contains(.,'\''clean'\'') and ' .../qa/workunits/ceph-helpers.sh:866: get_num_active_clean: expression+='not(contains(.,'\''stale'\''))' .../qa/workunits/ceph-helpers.sh:867: get_num_active_clean: expression+=')' .../qa/workunits/ceph-helpers.sh:871: get_num_active_clean: ceph --format xml pg dump pgs .../qa/workunits/ceph-helpers.sh:872: get_num_active_clean: xmlstarlet sel -t -m '//pg_stat/state[(contains(.,'\''active'\'') and contains(.,'\''clean'\'') and not(contains(.,'\''stale'\'')))]' -v . -n .../qa/workunits/ceph-helpers.sh:873: get_num_active_clean: grep -v '^$' .../qa/workunits/ceph-helpers.sh:873: get_num_active_clean: wc -l ../qa/workunits/ceph-helpers.sh:993: wait_for_clean: cur_active_clean=0 ../qa/workunits/ceph-helpers.sh:994: wait_for_clean: test 0 '!=' 0 ../qa/workunits/ceph-helpers.sh:998: wait_for_clean: sleep 1 .../qa/workunits/ceph-helpers.sh:999: wait_for_clean: expr 19 + 1 ../qa/workunits/ceph-helpers.sh:999: wait_for_clean: timer=20 ../qa/workunits/ceph-helpers.sh:985: wait_for_clean: is_clean .../qa/workunits/ceph-helpers.sh:952: is_clean: get_num_pgs .../qa/workunits/ceph-helpers.sh:898: get_num_pgs: ceph --format xml status .../qa/workunits/ceph-helpers.sh:899: get_num_pgs: xmlstarlet sel -t -m //pgmap/num_pgs -v . ../qa/workunits/ceph-helpers.sh:952: is_clean: num_pgs=4 ../qa/workunits/ceph-helpers.sh:953: is_clean: test 4 '!=' 0 .../qa/workunits/ceph-helpers.sh:954: is_clean: get_num_active_clean .../qa/workunits/ceph-helpers.sh:863: get_num_active_clean: local 'expression=(' .../qa/workunits/ceph-helpers.sh:864: get_num_active_clean: expression+='contains(.,'\''active'\'') and ' .../qa/workunits/ceph-helpers.sh:865: get_num_active_clean: expression+='contains(.,'\''clean'\'') and ' .../qa/workunits/ceph-helpers.sh:866: get_num_active_clean: expression+='not(contains(.,'\''stale'\''))' .../qa/workunits/ceph-helpers.sh:867: get_num_active_clean: expression+=')' .../qa/workunits/ceph-helpers.sh:871: get_num_active_clean: ceph --format xml pg dump pgs .../qa/workunits/ceph-helpers.sh:872: get_num_active_clean: xmlstarlet sel -t -m '//pg_stat/state[(contains(.,'\''active'\'') and contains(.,'\''clean'\'') and not(contains(.,'\''stale'\'')))]' -v . -n .../qa/workunits/ceph-helpers.sh:873: get_num_active_clean: wc -l .../qa/workunits/ceph-helpers.sh:873: get_num_active_clean: grep -v '^$' ../qa/workunits/ceph-helpers.sh:954: is_clean: test 0 = 4 ../qa/workunits/ceph-helpers.sh:954: is_clean: return 1 ../qa/workunits/ceph-helpers.sh:986: wait_for_clean: get_is_making_recovery_progress .../qa/workunits/ceph-helpers.sh:839: get_is_making_recovery_progress: ceph --format xml status .../qa/workunits/ceph-helpers.sh:839: get_is_making_recovery_progress: xmlstarlet sel -t -m //pgmap/recovering_keys_per_sec -v . -o ' ' -t -m //pgmap/recovering_bytes_per_sec -v . -o ' ' -t -m //pgmap/recovering_objects_per_sec -v . ../qa/workunits/ceph-helpers.sh:839: get_is_making_recovery_progress: local progress= ../qa/workunits/ceph-helpers.sh:840: get_is_making_recovery_progress: test -n '' ../qa/workunits/ceph-helpers.sh:988: wait_for_clean: (( timer >= 120 )) .../qa/workunits/ceph-helpers.sh:993: wait_for_clean: get_num_active_clean .../qa/workunits/ceph-helpers.sh:863: get_num_active_clean: local 'expression=(' .../qa/workunits/ceph-helpers.sh:864: get_num_active_clean: expression+='contains(.,'\''active'\'') and ' .../qa/workunits/ceph-helpers.sh:865: get_num_active_clean: expression+='contains(.,'\''clean'\'') and ' .../qa/workunits/ceph-helpers.sh:866: get_num_active_clean: expression+='not(contains(.,'\''stale'\''))' .../qa/workunits/ceph-helpers.sh:867: get_num_active_clean: expression+=')' .../qa/workunits/ceph-helpers.sh:873: get_num_active_clean: wc -l .../qa/workunits/ceph-helpers.sh:873: get_num_active_clean: grep -v '^$' .../qa/workunits/ceph-helpers.sh:872: get_num_active_clean: xmlstarlet sel -t -m '//pg_stat/state[(contains(.,'\''active'\'') and contains(.,'\''clean'\'') and not(contains(.,'\''stale'\'')))]' -v . -n .../qa/workunits/ceph-helpers.sh:871: get_num_active_clean: ceph --format xml pg dump pgs ../qa/workunits/ceph-helpers.sh:993: wait_for_clean: cur_active_clean=0 ../qa/workunits/ceph-helpers.sh:994: wait_for_clean: test 0 '!=' 0 ../qa/workunits/ceph-helpers.sh:998: wait_for_clean: sleep 1 .../qa/workunits/ceph-helpers.sh:999: wait_for_clean: expr 20 + 1 ../qa/workunits/ceph-helpers.sh:999: wait_for_clean: timer=21 ../qa/workunits/ceph-helpers.sh:985: wait_for_clean: is_clean .../qa/workunits/ceph-helpers.sh:952: is_clean: get_num_pgs .../qa/workunits/ceph-helpers.sh:898: get_num_pgs: ceph --format xml status .../qa/workunits/ceph-helpers.sh:899: get_num_pgs: xmlstarlet sel -t -m //pgmap/num_pgs -v . ../qa/workunits/ceph-helpers.sh:952: is_clean: num_pgs=4 ../qa/workunits/ceph-helpers.sh:953: is_clean: test 4 '!=' 0 .../qa/workunits/ceph-helpers.sh:954: is_clean: get_num_active_clean .../qa/workunits/ceph-helpers.sh:863: get_num_active_clean: local 'expression=(' .../qa/workunits/ceph-helpers.sh:864: get_num_active_clean: expression+='contains(.,'\''active'\'') and ' .../qa/workunits/ceph-helpers.sh:865: get_num_active_clean: expression+='contains(.,'\''clean'\'') and ' .../qa/workunits/ceph-helpers.sh:866: get_num_active_clean: expression+='not(contains(.,'\''stale'\''))' .../qa/workunits/ceph-helpers.sh:867: get_num_active_clean: expression+=')' .../qa/workunits/ceph-helpers.sh:871: get_num_active_clean: ceph --format xml pg dump pgs .../qa/workunits/ceph-helpers.sh:872: get_num_active_clean: xmlstarlet sel -t -m '//pg_stat/state[(contains(.,'\''active'\'') and contains(.,'\''clean'\'') and not(contains(.,'\''stale'\'')))]' -v . -n .../qa/workunits/ceph-helpers.sh:873: get_num_active_clean: wc -l .../qa/workunits/ceph-helpers.sh:873: get_num_active_clean: grep -v '^$' ../qa/workunits/ceph-helpers.sh:954: is_clean: test 0 = 4 ../qa/workunits/ceph-helpers.sh:954: is_clean: return 1 ../qa/workunits/ceph-helpers.sh:986: wait_for_clean: get_is_making_recovery_progress .../qa/workunits/ceph-helpers.sh:839: get_is_making_recovery_progress: ceph --format xml status .../qa/workunits/ceph-helpers.sh:839: get_is_making_recovery_progress: xmlstarlet sel -t -m //pgmap/recovering_keys_per_sec -v . -o ' ' -t -m //pgmap/recovering_bytes_per_sec -v . -o ' ' -t -m //pgmap/recovering_objects_per_sec -v . ../qa/workunits/ceph-helpers.sh:839: get_is_making_recovery_progress: local progress= ../qa/workunits/ceph-helpers.sh:840: get_is_making_recovery_progress: test -n '' ../qa/workunits/ceph-helpers.sh:988: wait_for_clean: (( timer >= 120 )) .../qa/workunits/ceph-helpers.sh:993: wait_for_clean: get_num_active_clean .../qa/workunits/ceph-helpers.sh:863: get_num_active_clean: local 'expression=(' .../qa/workunits/ceph-helpers.sh:864: get_num_active_clean: expression+='contains(.,'\''active'\'') and ' .../qa/workunits/ceph-helpers.sh:865: get_num_active_clean: expression+='contains(.,'\''clean'\'') and ' .../qa/workunits/ceph-helpers.sh:866: get_num_active_clean: expression+='not(contains(.,'\''stale'\''))' .../qa/workunits/ceph-helpers.sh:867: get_num_active_clean: expression+=')' .../qa/workunits/ceph-helpers.sh:871: get_num_active_clean: ceph --format xml pg dump pgs .../qa/workunits/ceph-helpers.sh:872: get_num_active_clean: xmlstarlet sel -t -m '//pg_stat/state[(contains(.,'\''active'\'') and contains(.,'\''clean'\'') and not(contains(.,'\''stale'\'')))]' -v . -n .../qa/workunits/ceph-helpers.sh:873: get_num_active_clean: wc -l .../qa/workunits/ceph-helpers.sh:873: get_num_active_clean: grep -v '^$' ../qa/workunits/ceph-helpers.sh:993: wait_for_clean: cur_active_clean=0 ../qa/workunits/ceph-helpers.sh:994: wait_for_clean: test 0 '!=' 0 ../qa/workunits/ceph-helpers.sh:998: wait_for_clean: sleep 1 .../qa/workunits/ceph-helpers.sh:999: wait_for_clean: expr 21 + 1 ../qa/workunits/ceph-helpers.sh:999: wait_for_clean: timer=22 ../qa/workunits/ceph-helpers.sh:985: wait_for_clean: is_clean .../qa/workunits/ceph-helpers.sh:952: is_clean: get_num_pgs .../qa/workunits/ceph-helpers.sh:898: get_num_pgs: ceph --format xml status .../qa/workunits/ceph-helpers.sh:899: get_num_pgs: xmlstarlet sel -t -m //pgmap/num_pgs -v . ../qa/workunits/ceph-helpers.sh:952: is_clean: num_pgs=4 ../qa/workunits/ceph-helpers.sh:953: is_clean: test 4 '!=' 0 .../qa/workunits/ceph-helpers.sh:954: is_clean: get_num_active_clean .../qa/workunits/ceph-helpers.sh:863: get_num_active_clean: local 'expression=(' .../qa/workunits/ceph-helpers.sh:864: get_num_active_clean: expression+='contains(.,'\''active'\'') and ' .../qa/workunits/ceph-helpers.sh:865: get_num_active_clean: expression+='contains(.,'\''clean'\'') and ' .../qa/workunits/ceph-helpers.sh:866: get_num_active_clean: expression+='not(contains(.,'\''stale'\''))' .../qa/workunits/ceph-helpers.sh:867: get_num_active_clean: expression+=')' .../qa/workunits/ceph-helpers.sh:871: get_num_active_clean: ceph --format xml pg dump pgs .../qa/workunits/ceph-helpers.sh:873: get_num_active_clean: grep -v '^$' .../qa/workunits/ceph-helpers.sh:872: get_num_active_clean: xmlstarlet sel -t -m '//pg_stat/state[(contains(.,'\''active'\'') and contains(.,'\''clean'\'') and not(contains(.,'\''stale'\'')))]' -v . -n .../qa/workunits/ceph-helpers.sh:873: get_num_active_clean: wc -l ../qa/workunits/ceph-helpers.sh:954: is_clean: test 0 = 4 ../qa/workunits/ceph-helpers.sh:954: is_clean: return 1 ../qa/workunits/ceph-helpers.sh:986: wait_for_clean: get_is_making_recovery_progress .../qa/workunits/ceph-helpers.sh:839: get_is_making_recovery_progress: ceph --format xml status .../qa/workunits/ceph-helpers.sh:839: get_is_making_recovery_progress: xmlstarlet sel -t -m //pgmap/recovering_keys_per_sec -v . -o ' ' -t -m //pgmap/recovering_bytes_per_sec -v . -o ' ' -t -m //pgmap/recovering_objects_per_sec -v . ../qa/workunits/ceph-helpers.sh:839: get_is_making_recovery_progress: local progress= ../qa/workunits/ceph-helpers.sh:840: get_is_making_recovery_progress: test -n '' ../qa/workunits/ceph-helpers.sh:988: wait_for_clean: (( timer >= 120 )) .../qa/workunits/ceph-helpers.sh:993: wait_for_clean: get_num_active_clean .../qa/workunits/ceph-helpers.sh:863: get_num_active_clean: local 'expression=(' .../qa/workunits/ceph-helpers.sh:864: get_num_active_clean: expression+='contains(.,'\''active'\'') and ' .../qa/workunits/ceph-helpers.sh:865: get_num_active_clean: expression+='contains(.,'\''clean'\'') and ' .../qa/workunits/ceph-helpers.sh:866: get_num_active_clean: expression+='not(contains(.,'\''stale'\''))' .../qa/workunits/ceph-helpers.sh:867: get_num_active_clean: expression+=')' .../qa/workunits/ceph-helpers.sh:871: get_num_active_clean: ceph --format xml pg dump pgs .../qa/workunits/ceph-helpers.sh:872: get_num_active_clean: xmlstarlet sel -t -m '//pg_stat/state[(contains(.,'\''active'\'') and contains(.,'\''clean'\'') and not(contains(.,'\''stale'\'')))]' -v . -n .../qa/workunits/ceph-helpers.sh:873: get_num_active_clean: grep -v '^$' .../qa/workunits/ceph-helpers.sh:873: get_num_active_clean: wc -l ../qa/workunits/ceph-helpers.sh:993: wait_for_clean: cur_active_clean=0 ../qa/workunits/ceph-helpers.sh:994: wait_for_clean: test 0 '!=' 0 ../qa/workunits/ceph-helpers.sh:998: wait_for_clean: sleep 1 .../qa/workunits/ceph-helpers.sh:999: wait_for_clean: expr 22 + 1 ../qa/workunits/ceph-helpers.sh:999: wait_for_clean: timer=23 ../qa/workunits/ceph-helpers.sh:985: wait_for_clean: is_clean .../qa/workunits/ceph-helpers.sh:952: is_clean: get_num_pgs .../qa/workunits/ceph-helpers.sh:899: get_num_pgs: xmlstarlet sel -t -m //pgmap/num_pgs -v . .../qa/workunits/ceph-helpers.sh:898: get_num_pgs: ceph --format xml status ../qa/workunits/ceph-helpers.sh:952: is_clean: num_pgs=4 ../qa/workunits/ceph-helpers.sh:953: is_clean: test 4 '!=' 0 .../qa/workunits/ceph-helpers.sh:954: is_clean: get_num_active_clean .../qa/workunits/ceph-helpers.sh:863: get_num_active_clean: local 'expression=(' .../qa/workunits/ceph-helpers.sh:864: get_num_active_clean: expression+='contains(.,'\''active'\'') and ' .../qa/workunits/ceph-helpers.sh:865: get_num_active_clean: expression+='contains(.,'\''clean'\'') and ' .../qa/workunits/ceph-helpers.sh:866: get_num_active_clean: expression+='not(contains(.,'\''stale'\''))' .../qa/workunits/ceph-helpers.sh:867: get_num_active_clean: expression+=')' .../qa/workunits/ceph-helpers.sh:871: get_num_active_clean: ceph --format xml pg dump pgs .../qa/workunits/ceph-helpers.sh:872: get_num_active_clean: xmlstarlet sel -t -m '//pg_stat/state[(contains(.,'\''active'\'') and contains(.,'\''clean'\'') and not(contains(.,'\''stale'\'')))]' -v . -n .../qa/workunits/ceph-helpers.sh:873: get_num_active_clean: grep -v '^$' .../qa/workunits/ceph-helpers.sh:873: get_num_active_clean: wc -l ../qa/workunits/ceph-helpers.sh:954: is_clean: test 0 = 4 ../qa/workunits/ceph-helpers.sh:954: is_clean: return 1 ../qa/workunits/ceph-helpers.sh:986: wait_for_clean: get_is_making_recovery_progress .../qa/workunits/ceph-helpers.sh:839: get_is_making_recovery_progress: ceph --format xml status .../qa/workunits/ceph-helpers.sh:839: get_is_making_recovery_progress: xmlstarlet sel -t -m //pgmap/recovering_keys_per_sec -v . -o ' ' -t -m //pgmap/recovering_bytes_per_sec -v . -o ' ' -t -m //pgmap/recovering_objects_per_sec -v . ../qa/workunits/ceph-helpers.sh:839: get_is_making_recovery_progress: local progress= ../qa/workunits/ceph-helpers.sh:840: get_is_making_recovery_progress: test -n '' ../qa/workunits/ceph-helpers.sh:988: wait_for_clean: (( timer >= 120 )) .../qa/workunits/ceph-helpers.sh:993: wait_for_clean: get_num_active_clean .../qa/workunits/ceph-helpers.sh:863: get_num_active_clean: local 'expression=(' .../qa/workunits/ceph-helpers.sh:864: get_num_active_clean: expression+='contains(.,'\''active'\'') and ' .../qa/workunits/ceph-helpers.sh:865: get_num_active_clean: expression+='contains(.,'\''clean'\'') and ' .../qa/workunits/ceph-helpers.sh:866: get_num_active_clean: expression+='not(contains(.,'\''stale'\''))' .../qa/workunits/ceph-helpers.sh:867: get_num_active_clean: expression+=')' .../qa/workunits/ceph-helpers.sh:871: get_num_active_clean: ceph --format xml pg dump pgs .../qa/workunits/ceph-helpers.sh:872: get_num_active_clean: xmlstarlet sel -t -m '//pg_stat/state[(contains(.,'\''active'\'') and contains(.,'\''clean'\'') and not(contains(.,'\''stale'\'')))]' -v . -n .../qa/workunits/ceph-helpers.sh:873: get_num_active_clean: grep -v '^$' .../qa/workunits/ceph-helpers.sh:873: get_num_active_clean: wc -l ../qa/workunits/ceph-helpers.sh:993: wait_for_clean: cur_active_clean=0 ../qa/workunits/ceph-helpers.sh:994: wait_for_clean: test 0 '!=' 0 ../qa/workunits/ceph-helpers.sh:998: wait_for_clean: sleep 1 .../qa/workunits/ceph-helpers.sh:999: wait_for_clean: expr 23 + 1 ../qa/workunits/ceph-helpers.sh:999: wait_for_clean: timer=24 ../qa/workunits/ceph-helpers.sh:985: wait_for_clean: is_clean .../qa/workunits/ceph-helpers.sh:952: is_clean: get_num_pgs .../qa/workunits/ceph-helpers.sh:898: get_num_pgs: ceph --format xml status .../qa/workunits/ceph-helpers.sh:899: get_num_pgs: xmlstarlet sel -t -m //pgmap/num_pgs -v . ../qa/workunits/ceph-helpers.sh:952: is_clean: num_pgs=4 ../qa/workunits/ceph-helpers.sh:953: is_clean: test 4 '!=' 0 .../qa/workunits/ceph-helpers.sh:954: is_clean: get_num_active_clean .../qa/workunits/ceph-helpers.sh:863: get_num_active_clean: local 'expression=(' .../qa/workunits/ceph-helpers.sh:864: get_num_active_clean: expression+='contains(.,'\''active'\'') and ' .../qa/workunits/ceph-helpers.sh:865: get_num_active_clean: expression+='contains(.,'\''clean'\'') and ' .../qa/workunits/ceph-helpers.sh:866: get_num_active_clean: expression+='not(contains(.,'\''stale'\''))' .../qa/workunits/ceph-helpers.sh:867: get_num_active_clean: expression+=')' .../qa/workunits/ceph-helpers.sh:871: get_num_active_clean: ceph --format xml pg dump pgs .../qa/workunits/ceph-helpers.sh:872: get_num_active_clean: xmlstarlet sel -t -m '//pg_stat/state[(contains(.,'\''active'\'') and contains(.,'\''clean'\'') and not(contains(.,'\''stale'\'')))]' -v . -n .../qa/workunits/ceph-helpers.sh:873: get_num_active_clean: grep -v '^$' .../qa/workunits/ceph-helpers.sh:873: get_num_active_clean: wc -l ../qa/workunits/ceph-helpers.sh:954: is_clean: test 0 = 4 ../qa/workunits/ceph-helpers.sh:954: is_clean: return 1 ../qa/workunits/ceph-helpers.sh:986: wait_for_clean: get_is_making_recovery_progress .../qa/workunits/ceph-helpers.sh:839: get_is_making_recovery_progress: ceph --format xml status .../qa/workunits/ceph-helpers.sh:839: get_is_making_recovery_progress: xmlstarlet sel -t -m //pgmap/recovering_keys_per_sec -v . -o ' ' -t -m //pgmap/recovering_bytes_per_sec -v . -o ' ' -t -m //pgmap/recovering_objects_per_sec -v . ../qa/workunits/ceph-helpers.sh:839: get_is_making_recovery_progress: local progress= ../qa/workunits/ceph-helpers.sh:840: get_is_making_recovery_progress: test -n '' ../qa/workunits/ceph-helpers.sh:988: wait_for_clean: (( timer >= 120 )) .../qa/workunits/ceph-helpers.sh:993: wait_for_clean: get_num_active_clean .../qa/workunits/ceph-helpers.sh:863: get_num_active_clean: local 'expression=(' .../qa/workunits/ceph-helpers.sh:864: get_num_active_clean: expression+='contains(.,'\''active'\'') and ' .../qa/workunits/ceph-helpers.sh:865: get_num_active_clean: expression+='contains(.,'\''clean'\'') and ' .../qa/workunits/ceph-helpers.sh:866: get_num_active_clean: expression+='not(contains(.,'\''stale'\''))' .../qa/workunits/ceph-helpers.sh:867: get_num_active_clean: expression+=')' .../qa/workunits/ceph-helpers.sh:871: get_num_active_clean: ceph --format xml pg dump pgs .../qa/workunits/ceph-helpers.sh:872: get_num_active_clean: xmlstarlet sel -t -m '//pg_stat/state[(contains(.,'\''active'\'') and contains(.,'\''clean'\'') and not(contains(.,'\''stale'\'')))]' -v . -n .../qa/workunits/ceph-helpers.sh:873: get_num_active_clean: wc -l .../qa/workunits/ceph-helpers.sh:873: get_num_active_clean: grep -v '^$' ../qa/workunits/ceph-helpers.sh:993: wait_for_clean: cur_active_clean=0 ../qa/workunits/ceph-helpers.sh:994: wait_for_clean: test 0 '!=' 0 ../qa/workunits/ceph-helpers.sh:998: wait_for_clean: sleep 1 .../qa/workunits/ceph-helpers.sh:999: wait_for_clean: expr 24 + 1 ../qa/workunits/ceph-helpers.sh:999: wait_for_clean: timer=25 ../qa/workunits/ceph-helpers.sh:985: wait_for_clean: is_clean .../qa/workunits/ceph-helpers.sh:952: is_clean: get_num_pgs .../qa/workunits/ceph-helpers.sh:898: get_num_pgs: ceph --format xml status .../qa/workunits/ceph-helpers.sh:899: get_num_pgs: xmlstarlet sel -t -m //pgmap/num_pgs -v . ../qa/workunits/ceph-helpers.sh:952: is_clean: num_pgs=4 ../qa/workunits/ceph-helpers.sh:953: is_clean: test 4 '!=' 0 .../qa/workunits/ceph-helpers.sh:954: is_clean: get_num_active_clean .../qa/workunits/ceph-helpers.sh:863: get_num_active_clean: local 'expression=(' .../qa/workunits/ceph-helpers.sh:864: get_num_active_clean: expression+='contains(.,'\''active'\'') and ' .../qa/workunits/ceph-helpers.sh:865: get_num_active_clean: expression+='contains(.,'\''clean'\'') and ' .../qa/workunits/ceph-helpers.sh:866: get_num_active_clean: expression+='not(contains(.,'\''stale'\''))' .../qa/workunits/ceph-helpers.sh:867: get_num_active_clean: expression+=')' .../qa/workunits/ceph-helpers.sh:871: get_num_active_clean: ceph --format xml pg dump pgs .../qa/workunits/ceph-helpers.sh:872: get_num_active_clean: xmlstarlet sel -t -m '//pg_stat/state[(contains(.,'\''active'\'') and contains(.,'\''clean'\'') and not(contains(.,'\''stale'\'')))]' -v . -n .../qa/workunits/ceph-helpers.sh:873: get_num_active_clean: grep -v '^$' .../qa/workunits/ceph-helpers.sh:873: get_num_active_clean: wc -l ../qa/workunits/ceph-helpers.sh:954: is_clean: test 0 = 4 ../qa/workunits/ceph-helpers.sh:954: is_clean: return 1 ../qa/workunits/ceph-helpers.sh:986: wait_for_clean: get_is_making_recovery_progress .../qa/workunits/ceph-helpers.sh:839: get_is_making_recovery_progress: ceph --format xml status .../qa/workunits/ceph-helpers.sh:839: get_is_making_recovery_progress: xmlstarlet sel -t -m //pgmap/recovering_keys_per_sec -v . -o ' ' -t -m //pgmap/recovering_bytes_per_sec -v . -o ' ' -t -m //pgmap/recovering_objects_per_sec -v . ../qa/workunits/ceph-helpers.sh:839: get_is_making_recovery_progress: local progress= ../qa/workunits/ceph-helpers.sh:840: get_is_making_recovery_progress: test -n '' ../qa/workunits/ceph-helpers.sh:988: wait_for_clean: (( timer >= 120 )) .../qa/workunits/ceph-helpers.sh:993: wait_for_clean: get_num_active_clean .../qa/workunits/ceph-helpers.sh:863: get_num_active_clean: local 'expression=(' .../qa/workunits/ceph-helpers.sh:864: get_num_active_clean: expression+='contains(.,'\''active'\'') and ' .../qa/workunits/ceph-helpers.sh:865: get_num_active_clean: expression+='contains(.,'\''clean'\'') and ' .../qa/workunits/ceph-helpers.sh:866: get_num_active_clean: expression+='not(contains(.,'\''stale'\''))' .../qa/workunits/ceph-helpers.sh:867: get_num_active_clean: expression+=')' .../qa/workunits/ceph-helpers.sh:871: get_num_active_clean: ceph --format xml pg dump pgs .../qa/workunits/ceph-helpers.sh:873: get_num_active_clean: wc -l .../qa/workunits/ceph-helpers.sh:872: get_num_active_clean: xmlstarlet sel -t -m '//pg_stat/state[(contains(.,'\''active'\'') and contains(.,'\''clean'\'') and not(contains(.,'\''stale'\'')))]' -v . -n .../qa/workunits/ceph-helpers.sh:873: get_num_active_clean: grep -v '^$' ../qa/workunits/ceph-helpers.sh:993: wait_for_clean: cur_active_clean=0 ../qa/workunits/ceph-helpers.sh:994: wait_for_clean: test 0 '!=' 0 ../qa/workunits/ceph-helpers.sh:998: wait_for_clean: sleep 1 .../qa/workunits/ceph-helpers.sh:999: wait_for_clean: expr 25 + 1 ../qa/workunits/ceph-helpers.sh:999: wait_for_clean: timer=26 ../qa/workunits/ceph-helpers.sh:985: wait_for_clean: is_clean .../qa/workunits/ceph-helpers.sh:952: is_clean: get_num_pgs .../qa/workunits/ceph-helpers.sh:898: get_num_pgs: ceph --format xml status .../qa/workunits/ceph-helpers.sh:899: get_num_pgs: xmlstarlet sel -t -m //pgmap/num_pgs -v . ../qa/workunits/ceph-helpers.sh:952: is_clean: num_pgs=4 ../qa/workunits/ceph-helpers.sh:953: is_clean: test 4 '!=' 0 .../qa/workunits/ceph-helpers.sh:954: is_clean: get_num_active_clean .../qa/workunits/ceph-helpers.sh:863: get_num_active_clean: local 'expression=(' .../qa/workunits/ceph-helpers.sh:864: get_num_active_clean: expression+='contains(.,'\''active'\'') and ' .../qa/workunits/ceph-helpers.sh:865: get_num_active_clean: expression+='contains(.,'\''clean'\'') and ' .../qa/workunits/ceph-helpers.sh:866: get_num_active_clean: expression+='not(contains(.,'\''stale'\''))' .../qa/workunits/ceph-helpers.sh:867: get_num_active_clean: expression+=')' .../qa/workunits/ceph-helpers.sh:872: get_num_active_clean: xmlstarlet sel -t -m '//pg_stat/state[(contains(.,'\''active'\'') and contains(.,'\''clean'\'') and not(contains(.,'\''stale'\'')))]' -v . -n .../qa/workunits/ceph-helpers.sh:873: get_num_active_clean: grep -v '^$' .../qa/workunits/ceph-helpers.sh:873: get_num_active_clean: wc -l .../qa/workunits/ceph-helpers.sh:871: get_num_active_clean: ceph --format xml pg dump pgs ../qa/workunits/ceph-helpers.sh:954: is_clean: test 0 = 4 ../qa/workunits/ceph-helpers.sh:954: is_clean: return 1 ../qa/workunits/ceph-helpers.sh:986: wait_for_clean: get_is_making_recovery_progress .../qa/workunits/ceph-helpers.sh:839: get_is_making_recovery_progress: ceph --format xml status .../qa/workunits/ceph-helpers.sh:839: get_is_making_recovery_progress: xmlstarlet sel -t -m //pgmap/recovering_keys_per_sec -v . -o ' ' -t -m //pgmap/recovering_bytes_per_sec -v . -o ' ' -t -m //pgmap/recovering_objects_per_sec -v . ../qa/workunits/ceph-helpers.sh:839: get_is_making_recovery_progress: local progress= ../qa/workunits/ceph-helpers.sh:840: get_is_making_recovery_progress: test -n '' ../qa/workunits/ceph-helpers.sh:988: wait_for_clean: (( timer >= 120 )) .../qa/workunits/ceph-helpers.sh:993: wait_for_clean: get_num_active_clean .../qa/workunits/ceph-helpers.sh:863: get_num_active_clean: local 'expression=(' .../qa/workunits/ceph-helpers.sh:864: get_num_active_clean: expression+='contains(.,'\''active'\'') and ' .../qa/workunits/ceph-helpers.sh:865: get_num_active_clean: expression+='contains(.,'\''clean'\'') and ' .../qa/workunits/ceph-helpers.sh:866: get_num_active_clean: expression+='not(contains(.,'\''stale'\''))' .../qa/workunits/ceph-helpers.sh:867: get_num_active_clean: expression+=')' .../qa/workunits/ceph-helpers.sh:871: get_num_active_clean: ceph --format xml pg dump pgs .../qa/workunits/ceph-helpers.sh:872: get_num_active_clean: xmlstarlet sel -t -m '//pg_stat/state[(contains(.,'\''active'\'') and contains(.,'\''clean'\'') and not(contains(.,'\''stale'\'')))]' -v . -n .../qa/workunits/ceph-helpers.sh:873: get_num_active_clean: grep -v '^$' .../qa/workunits/ceph-helpers.sh:873: get_num_active_clean: wc -l ../qa/workunits/ceph-helpers.sh:993: wait_for_clean: cur_active_clean=0 ../qa/workunits/ceph-helpers.sh:994: wait_for_clean: test 0 '!=' 0 ../qa/workunits/ceph-helpers.sh:998: wait_for_clean: sleep 1 .../qa/workunits/ceph-helpers.sh:999: wait_for_clean: expr 26 + 1 ../qa/workunits/ceph-helpers.sh:999: wait_for_clean: timer=27 ../qa/workunits/ceph-helpers.sh:985: wait_for_clean: is_clean .../qa/workunits/ceph-helpers.sh:952: is_clean: get_num_pgs .../qa/workunits/ceph-helpers.sh:898: get_num_pgs: ceph --format xml status .../qa/workunits/ceph-helpers.sh:899: get_num_pgs: xmlstarlet sel -t -m //pgmap/num_pgs -v . ../qa/workunits/ceph-helpers.sh:952: is_clean: num_pgs=4 ../qa/workunits/ceph-helpers.sh:953: is_clean: test 4 '!=' 0 .../qa/workunits/ceph-helpers.sh:954: is_clean: get_num_active_clean .../qa/workunits/ceph-helpers.sh:863: get_num_active_clean: local 'expression=(' .../qa/workunits/ceph-helpers.sh:864: get_num_active_clean: expression+='contains(.,'\''active'\'') and ' .../qa/workunits/ceph-helpers.sh:865: get_num_active_clean: expression+='contains(.,'\''clean'\'') and ' .../qa/workunits/ceph-helpers.sh:866: get_num_active_clean: expression+='not(contains(.,'\''stale'\''))' .../qa/workunits/ceph-helpers.sh:867: get_num_active_clean: expression+=')' .../qa/workunits/ceph-helpers.sh:871: get_num_active_clean: ceph --format xml pg dump pgs .../qa/workunits/ceph-helpers.sh:872: get_num_active_clean: xmlstarlet sel -t -m '//pg_stat/state[(contains(.,'\''active'\'') and contains(.,'\''clean'\'') and not(contains(.,'\''stale'\'')))]' -v . -n .../qa/workunits/ceph-helpers.sh:873: get_num_active_clean: grep -v '^$' .../qa/workunits/ceph-helpers.sh:873: get_num_active_clean: wc -l ../qa/workunits/ceph-helpers.sh:954: is_clean: test 0 = 4 ../qa/workunits/ceph-helpers.sh:954: is_clean: return 1 ../qa/workunits/ceph-helpers.sh:986: wait_for_clean: get_is_making_recovery_progress .../qa/workunits/ceph-helpers.sh:839: get_is_making_recovery_progress: ceph --format xml status .../qa/workunits/ceph-helpers.sh:839: get_is_making_recovery_progress: xmlstarlet sel -t -m //pgmap/recovering_keys_per_sec -v . -o ' ' -t -m //pgmap/recovering_bytes_per_sec -v . -o ' ' -t -m //pgmap/recovering_objects_per_sec -v . ../qa/workunits/ceph-helpers.sh:839: get_is_making_recovery_progress: local progress= ../qa/workunits/ceph-helpers.sh:840: get_is_making_recovery_progress: test -n '' ../qa/workunits/ceph-helpers.sh:988: wait_for_clean: (( timer >= 120 )) .../qa/workunits/ceph-helpers.sh:993: wait_for_clean: get_num_active_clean .../qa/workunits/ceph-helpers.sh:863: get_num_active_clean: local 'expression=(' .../qa/workunits/ceph-helpers.sh:864: get_num_active_clean: expression+='contains(.,'\''active'\'') and ' .../qa/workunits/ceph-helpers.sh:865: get_num_active_clean: expression+='contains(.,'\''clean'\'') and ' .../qa/workunits/ceph-helpers.sh:866: get_num_active_clean: expression+='not(contains(.,'\''stale'\''))' .../qa/workunits/ceph-helpers.sh:867: get_num_active_clean: expression+=')' .../qa/workunits/ceph-helpers.sh:871: get_num_active_clean: ceph --format xml pg dump pgs .../qa/workunits/ceph-helpers.sh:873: get_num_active_clean: grep -v '^$' .../qa/workunits/ceph-helpers.sh:873: get_num_active_clean: wc -l .../qa/workunits/ceph-helpers.sh:872: get_num_active_clean: xmlstarlet sel -t -m '//pg_stat/state[(contains(.,'\''active'\'') and contains(.,'\''clean'\'') and not(contains(.,'\''stale'\'')))]' -v . -n ../qa/workunits/ceph-helpers.sh:993: wait_for_clean: cur_active_clean=0 ../qa/workunits/ceph-helpers.sh:994: wait_for_clean: test 0 '!=' 0 ../qa/workunits/ceph-helpers.sh:998: wait_for_clean: sleep 1 .../qa/workunits/ceph-helpers.sh:999: wait_for_clean: expr 27 + 1 ../qa/workunits/ceph-helpers.sh:999: wait_for_clean: timer=28 ../qa/workunits/ceph-helpers.sh:985: wait_for_clean: is_clean .../qa/workunits/ceph-helpers.sh:952: is_clean: get_num_pgs .../qa/workunits/ceph-helpers.sh:898: get_num_pgs: ceph --format xml status .../qa/workunits/ceph-helpers.sh:899: get_num_pgs: xmlstarlet sel -t -m //pgmap/num_pgs -v . ../qa/workunits/ceph-helpers.sh:952: is_clean: num_pgs=4 ../qa/workunits/ceph-helpers.sh:953: is_clean: test 4 '!=' 0 .../qa/workunits/ceph-helpers.sh:954: is_clean: get_num_active_clean .../qa/workunits/ceph-helpers.sh:863: get_num_active_clean: local 'expression=(' .../qa/workunits/ceph-helpers.sh:864: get_num_active_clean: expression+='contains(.,'\''active'\'') and ' .../qa/workunits/ceph-helpers.sh:865: get_num_active_clean: expression+='contains(.,'\''clean'\'') and ' .../qa/workunits/ceph-helpers.sh:866: get_num_active_clean: expression+='not(contains(.,'\''stale'\''))' .../qa/workunits/ceph-helpers.sh:867: get_num_active_clean: expression+=')' .../qa/workunits/ceph-helpers.sh:871: get_num_active_clean: ceph --format xml pg dump pgs .../qa/workunits/ceph-helpers.sh:872: get_num_active_clean: xmlstarlet sel -t -m '//pg_stat/state[(contains(.,'\''active'\'') and contains(.,'\''clean'\'') and not(contains(.,'\''stale'\'')))]' -v . -n .../qa/workunits/ceph-helpers.sh:873: get_num_active_clean: wc -l .../qa/workunits/ceph-helpers.sh:873: get_num_active_clean: grep -v '^$' ../qa/workunits/ceph-helpers.sh:954: is_clean: test 0 = 4 ../qa/workunits/ceph-helpers.sh:954: is_clean: return 1 ../qa/workunits/ceph-helpers.sh:986: wait_for_clean: get_is_making_recovery_progress .../qa/workunits/ceph-helpers.sh:839: get_is_making_recovery_progress: ceph --format xml status .../qa/workunits/ceph-helpers.sh:839: get_is_making_recovery_progress: xmlstarlet sel -t -m //pgmap/recovering_keys_per_sec -v . -o ' ' -t -m //pgmap/recovering_bytes_per_sec -v . -o ' ' -t -m //pgmap/recovering_objects_per_sec -v . ../qa/workunits/ceph-helpers.sh:839: get_is_making_recovery_progress: local progress= ../qa/workunits/ceph-helpers.sh:840: get_is_making_recovery_progress: test -n '' ../qa/workunits/ceph-helpers.sh:988: wait_for_clean: (( timer >= 120 )) .../qa/workunits/ceph-helpers.sh:993: wait_for_clean: get_num_active_clean .../qa/workunits/ceph-helpers.sh:863: get_num_active_clean: local 'expression=(' .../qa/workunits/ceph-helpers.sh:864: get_num_active_clean: expression+='contains(.,'\''active'\'') and ' .../qa/workunits/ceph-helpers.sh:865: get_num_active_clean: expression+='contains(.,'\''clean'\'') and ' .../qa/workunits/ceph-helpers.sh:866: get_num_active_clean: expression+='not(contains(.,'\''stale'\''))' .../qa/workunits/ceph-helpers.sh:867: get_num_active_clean: expression+=')' .../qa/workunits/ceph-helpers.sh:871: get_num_active_clean: ceph --format xml pg dump pgs .../qa/workunits/ceph-helpers.sh:873: get_num_active_clean: grep -v '^$' .../qa/workunits/ceph-helpers.sh:872: get_num_active_clean: xmlstarlet sel -t -m '//pg_stat/state[(contains(.,'\''active'\'') and contains(.,'\''clean'\'') and not(contains(.,'\''stale'\'')))]' -v . -n .../qa/workunits/ceph-helpers.sh:873: get_num_active_clean: wc -l ../qa/workunits/ceph-helpers.sh:993: wait_for_clean: cur_active_clean=0 ../qa/workunits/ceph-helpers.sh:994: wait_for_clean: test 0 '!=' 0 ../qa/workunits/ceph-helpers.sh:998: wait_for_clean: sleep 1 .../qa/workunits/ceph-helpers.sh:999: wait_for_clean: expr 28 + 1 ../qa/workunits/ceph-helpers.sh:999: wait_for_clean: timer=29 ../qa/workunits/ceph-helpers.sh:985: wait_for_clean: is_clean .../qa/workunits/ceph-helpers.sh:952: is_clean: get_num_pgs .../qa/workunits/ceph-helpers.sh:898: get_num_pgs: ceph --format xml status .../qa/workunits/ceph-helpers.sh:899: get_num_pgs: xmlstarlet sel -t -m //pgmap/num_pgs -v . ../qa/workunits/ceph-helpers.sh:952: is_clean: num_pgs=4 ../qa/workunits/ceph-helpers.sh:953: is_clean: test 4 '!=' 0 .../qa/workunits/ceph-helpers.sh:954: is_clean: get_num_active_clean .../qa/workunits/ceph-helpers.sh:863: get_num_active_clean: local 'expression=(' .../qa/workunits/ceph-helpers.sh:864: get_num_active_clean: expression+='contains(.,'\''active'\'') and ' .../qa/workunits/ceph-helpers.sh:865: get_num_active_clean: expression+='contains(.,'\''clean'\'') and ' .../qa/workunits/ceph-helpers.sh:866: get_num_active_clean: expression+='not(contains(.,'\''stale'\''))' .../qa/workunits/ceph-helpers.sh:867: get_num_active_clean: expression+=')' .../qa/workunits/ceph-helpers.sh:871: get_num_active_clean: ceph --format xml pg dump pgs .../qa/workunits/ceph-helpers.sh:872: get_num_active_clean: xmlstarlet sel -t -m '//pg_stat/state[(contains(.,'\''active'\'') and contains(.,'\''clean'\'') and not(contains(.,'\''stale'\'')))]' -v . -n .../qa/workunits/ceph-helpers.sh:873: get_num_active_clean: grep -v '^$' .../qa/workunits/ceph-helpers.sh:873: get_num_active_clean: wc -l ../qa/workunits/ceph-helpers.sh:954: is_clean: test 0 = 4 ../qa/workunits/ceph-helpers.sh:954: is_clean: return 1 ../qa/workunits/ceph-helpers.sh:986: wait_for_clean: get_is_making_recovery_progress .../qa/workunits/ceph-helpers.sh:839: get_is_making_recovery_progress: ceph --format xml status .../qa/workunits/ceph-helpers.sh:839: get_is_making_recovery_progress: xmlstarlet sel -t -m //pgmap/recovering_keys_per_sec -v . -o ' ' -t -m //pgmap/recovering_bytes_per_sec -v . -o ' ' -t -m //pgmap/recovering_objects_per_sec -v . ../qa/workunits/ceph-helpers.sh:839: get_is_making_recovery_progress: local progress= ../qa/workunits/ceph-helpers.sh:840: get_is_making_recovery_progress: test -n '' ../qa/workunits/ceph-helpers.sh:988: wait_for_clean: (( timer >= 120 )) .../qa/workunits/ceph-helpers.sh:993: wait_for_clean: get_num_active_clean .../qa/workunits/ceph-helpers.sh:863: get_num_active_clean: local 'expression=(' .../qa/workunits/ceph-helpers.sh:864: get_num_active_clean: expression+='contains(.,'\''active'\'') and ' .../qa/workunits/ceph-helpers.sh:865: get_num_active_clean: expression+='contains(.,'\''clean'\'') and ' .../qa/workunits/ceph-helpers.sh:866: get_num_active_clean: expression+='not(contains(.,'\''stale'\''))' .../qa/workunits/ceph-helpers.sh:867: get_num_active_clean: expression+=')' .../qa/workunits/ceph-helpers.sh:871: get_num_active_clean: ceph --format xml pg dump pgs .../qa/workunits/ceph-helpers.sh:872: get_num_active_clean: xmlstarlet sel -t -m '//pg_stat/state[(contains(.,'\''active'\'') and contains(.,'\''clean'\'') and not(contains(.,'\''stale'\'')))]' -v . -n .../qa/workunits/ceph-helpers.sh:873: get_num_active_clean: grep -v '^$' .../qa/workunits/ceph-helpers.sh:873: get_num_active_clean: wc -l ../qa/workunits/ceph-helpers.sh:993: wait_for_clean: cur_active_clean=0 ../qa/workunits/ceph-helpers.sh:994: wait_for_clean: test 0 '!=' 0 ../qa/workunits/ceph-helpers.sh:998: wait_for_clean: sleep 1 .../qa/workunits/ceph-helpers.sh:999: wait_for_clean: expr 29 + 1 ../qa/workunits/ceph-helpers.sh:999: wait_for_clean: timer=30 ../qa/workunits/ceph-helpers.sh:985: wait_for_clean: is_clean .../qa/workunits/ceph-helpers.sh:952: is_clean: get_num_pgs .../qa/workunits/ceph-helpers.sh:898: get_num_pgs: ceph --format xml status .../qa/workunits/ceph-helpers.sh:899: get_num_pgs: xmlstarlet sel -t -m //pgmap/num_pgs -v . ../qa/workunits/ceph-helpers.sh:952: is_clean: num_pgs=4 ../qa/workunits/ceph-helpers.sh:953: is_clean: test 4 '!=' 0 .../qa/workunits/ceph-helpers.sh:954: is_clean: get_num_active_clean .../qa/workunits/ceph-helpers.sh:863: get_num_active_clean: local 'expression=(' .../qa/workunits/ceph-helpers.sh:864: get_num_active_clean: expression+='contains(.,'\''active'\'') and ' .../qa/workunits/ceph-helpers.sh:865: get_num_active_clean: expression+='contains(.,'\''clean'\'') and ' .../qa/workunits/ceph-helpers.sh:866: get_num_active_clean: expression+='not(contains(.,'\''stale'\''))' .../qa/workunits/ceph-helpers.sh:867: get_num_active_clean: expression+=')' .../qa/workunits/ceph-helpers.sh:871: get_num_active_clean: ceph --format xml pg dump pgs .../qa/workunits/ceph-helpers.sh:872: get_num_active_clean: xmlstarlet sel -t -m '//pg_stat/state[(contains(.,'\''active'\'') and contains(.,'\''clean'\'') and not(contains(.,'\''stale'\'')))]' -v . -n .../qa/workunits/ceph-helpers.sh:873: get_num_active_clean: grep -v '^$' .../qa/workunits/ceph-helpers.sh:873: get_num_active_clean: wc -l ../qa/workunits/ceph-helpers.sh:954: is_clean: test 0 = 4 ../qa/workunits/ceph-helpers.sh:954: is_clean: return 1 ../qa/workunits/ceph-helpers.sh:986: wait_for_clean: get_is_making_recovery_progress .../qa/workunits/ceph-helpers.sh:839: get_is_making_recovery_progress: ceph --format xml status .../qa/workunits/ceph-helpers.sh:839: get_is_making_recovery_progress: xmlstarlet sel -t -m //pgmap/recovering_keys_per_sec -v . -o ' ' -t -m //pgmap/recovering_bytes_per_sec -v . -o ' ' -t -m //pgmap/recovering_objects_per_sec -v . ../qa/workunits/ceph-helpers.sh:839: get_is_making_recovery_progress: local progress= ../qa/workunits/ceph-helpers.sh:840: get_is_making_recovery_progress: test -n '' ../qa/workunits/ceph-helpers.sh:988: wait_for_clean: (( timer >= 120 )) .../qa/workunits/ceph-helpers.sh:993: wait_for_clean: get_num_active_clean .../qa/workunits/ceph-helpers.sh:863: get_num_active_clean: local 'expression=(' .../qa/workunits/ceph-helpers.sh:864: get_num_active_clean: expression+='contains(.,'\''active'\'') and ' .../qa/workunits/ceph-helpers.sh:865: get_num_active_clean: expression+='contains(.,'\''clean'\'') and ' .../qa/workunits/ceph-helpers.sh:866: get_num_active_clean: expression+='not(contains(.,'\''stale'\''))' .../qa/workunits/ceph-helpers.sh:867: get_num_active_clean: expression+=')' .../qa/workunits/ceph-helpers.sh:871: get_num_active_clean: ceph --format xml pg dump pgs .../qa/workunits/ceph-helpers.sh:872: get_num_active_clean: xmlstarlet sel -t -m '//pg_stat/state[(contains(.,'\''active'\'') and contains(.,'\''clean'\'') and not(contains(.,'\''stale'\'')))]' -v . -n .../qa/workunits/ceph-helpers.sh:873: get_num_active_clean: grep -v '^$' .../qa/workunits/ceph-helpers.sh:873: get_num_active_clean: wc -l ../qa/workunits/ceph-helpers.sh:993: wait_for_clean: cur_active_clean=0 ../qa/workunits/ceph-helpers.sh:994: wait_for_clean: test 0 '!=' 0 ../qa/workunits/ceph-helpers.sh:998: wait_for_clean: sleep 1 .../qa/workunits/ceph-helpers.sh:999: wait_for_clean: expr 30 + 1 ../qa/workunits/ceph-helpers.sh:999: wait_for_clean: timer=31 ../qa/workunits/ceph-helpers.sh:985: wait_for_clean: is_clean .../qa/workunits/ceph-helpers.sh:952: is_clean: get_num_pgs .../qa/workunits/ceph-helpers.sh:898: get_num_pgs: ceph --format xml status .../qa/workunits/ceph-helpers.sh:899: get_num_pgs: xmlstarlet sel -t -m //pgmap/num_pgs -v . ../qa/workunits/ceph-helpers.sh:952: is_clean: num_pgs=4 ../qa/workunits/ceph-helpers.sh:953: is_clean: test 4 '!=' 0 .../qa/workunits/ceph-helpers.sh:954: is_clean: get_num_active_clean .../qa/workunits/ceph-helpers.sh:863: get_num_active_clean: local 'expression=(' .../qa/workunits/ceph-helpers.sh:864: get_num_active_clean: expression+='contains(.,'\''active'\'') and ' .../qa/workunits/ceph-helpers.sh:865: get_num_active_clean: expression+='contains(.,'\''clean'\'') and ' .../qa/workunits/ceph-helpers.sh:866: get_num_active_clean: expression+='not(contains(.,'\''stale'\''))' .../qa/workunits/ceph-helpers.sh:867: get_num_active_clean: expression+=')' .../qa/workunits/ceph-helpers.sh:871: get_num_active_clean: ceph --format xml pg dump pgs .../qa/workunits/ceph-helpers.sh:872: get_num_active_clean: xmlstarlet sel -t -m '//pg_stat/state[(contains(.,'\''active'\'') and contains(.,'\''clean'\'') and not(contains(.,'\''stale'\'')))]' -v . -n .../qa/workunits/ceph-helpers.sh:873: get_num_active_clean: grep -v '^$' .../qa/workunits/ceph-helpers.sh:873: get_num_active_clean: wc -l ../qa/workunits/ceph-helpers.sh:954: is_clean: test 0 = 4 ../qa/workunits/ceph-helpers.sh:954: is_clean: return 1 ../qa/workunits/ceph-helpers.sh:986: wait_for_clean: get_is_making_recovery_progress .../qa/workunits/ceph-helpers.sh:839: get_is_making_recovery_progress: ceph --format xml status .../qa/workunits/ceph-helpers.sh:839: get_is_making_recovery_progress: xmlstarlet sel -t -m //pgmap/recovering_keys_per_sec -v . -o ' ' -t -m //pgmap/recovering_bytes_per_sec -v . -o ' ' -t -m //pgmap/recovering_objects_per_sec -v . ../qa/workunits/ceph-helpers.sh:839: get_is_making_recovery_progress: local progress= ../qa/workunits/ceph-helpers.sh:840: get_is_making_recovery_progress: test -n '' ../qa/workunits/ceph-helpers.sh:988: wait_for_clean: (( timer >= 120 )) .../qa/workunits/ceph-helpers.sh:993: wait_for_clean: get_num_active_clean .../qa/workunits/ceph-helpers.sh:863: get_num_active_clean: local 'expression=(' .../qa/workunits/ceph-helpers.sh:864: get_num_active_clean: expression+='contains(.,'\''active'\'') and ' .../qa/workunits/ceph-helpers.sh:865: get_num_active_clean: expression+='contains(.,'\''clean'\'') and ' .../qa/workunits/ceph-helpers.sh:866: get_num_active_clean: expression+='not(contains(.,'\''stale'\''))' .../qa/workunits/ceph-helpers.sh:867: get_num_active_clean: expression+=')' .../qa/workunits/ceph-helpers.sh:871: get_num_active_clean: ceph --format xml pg dump pgs .../qa/workunits/ceph-helpers.sh:873: get_num_active_clean: grep -v '^$' .../qa/workunits/ceph-helpers.sh:872: get_num_active_clean: xmlstarlet sel -t -m '//pg_stat/state[(contains(.,'\''active'\'') and contains(.,'\''clean'\'') and not(contains(.,'\''stale'\'')))]' -v . -n .../qa/workunits/ceph-helpers.sh:873: get_num_active_clean: wc -l ../qa/workunits/ceph-helpers.sh:993: wait_for_clean: cur_active_clean=0 ../qa/workunits/ceph-helpers.sh:994: wait_for_clean: test 0 '!=' 0 ../qa/workunits/ceph-helpers.sh:998: wait_for_clean: sleep 1 .../qa/workunits/ceph-helpers.sh:999: wait_for_clean: expr 31 + 1 ../qa/workunits/ceph-helpers.sh:999: wait_for_clean: timer=32 ../qa/workunits/ceph-helpers.sh:985: wait_for_clean: is_clean .../qa/workunits/ceph-helpers.sh:952: is_clean: get_num_pgs .../qa/workunits/ceph-helpers.sh:898: get_num_pgs: ceph --format xml status .../qa/workunits/ceph-helpers.sh:899: get_num_pgs: xmlstarlet sel -t -m //pgmap/num_pgs -v . ../qa/workunits/ceph-helpers.sh:952: is_clean: num_pgs=4 ../qa/workunits/ceph-helpers.sh:953: is_clean: test 4 '!=' 0 .../qa/workunits/ceph-helpers.sh:954: is_clean: get_num_active_clean .../qa/workunits/ceph-helpers.sh:863: get_num_active_clean: local 'expression=(' .../qa/workunits/ceph-helpers.sh:864: get_num_active_clean: expression+='contains(.,'\''active'\'') and ' .../qa/workunits/ceph-helpers.sh:865: get_num_active_clean: expression+='contains(.,'\''clean'\'') and ' .../qa/workunits/ceph-helpers.sh:866: get_num_active_clean: expression+='not(contains(.,'\''stale'\''))' .../qa/workunits/ceph-helpers.sh:867: get_num_active_clean: expression+=')' .../qa/workunits/ceph-helpers.sh:871: get_num_active_clean: ceph --format xml pg dump pgs .../qa/workunits/ceph-helpers.sh:872: get_num_active_clean: xmlstarlet sel -t -m '//pg_stat/state[(contains(.,'\''active'\'') and contains(.,'\''clean'\'') and not(contains(.,'\''stale'\'')))]' -v . -n .../qa/workunits/ceph-helpers.sh:873: get_num_active_clean: wc -l .../qa/workunits/ceph-helpers.sh:873: get_num_active_clean: grep -v '^$' ../qa/workunits/ceph-helpers.sh:954: is_clean: test 0 = 4 ../qa/workunits/ceph-helpers.sh:954: is_clean: return 1 ../qa/workunits/ceph-helpers.sh:986: wait_for_clean: get_is_making_recovery_progress .../qa/workunits/ceph-helpers.sh:839: get_is_making_recovery_progress: ceph --format xml status .../qa/workunits/ceph-helpers.sh:839: get_is_making_recovery_progress: xmlstarlet sel -t -m //pgmap/recovering_keys_per_sec -v . -o ' ' -t -m //pgmap/recovering_bytes_per_sec -v . -o ' ' -t -m //pgmap/recovering_objects_per_sec -v . ../qa/workunits/ceph-helpers.sh:839: get_is_making_recovery_progress: local progress= ../qa/workunits/ceph-helpers.sh:840: get_is_making_recovery_progress: test -n '' ../qa/workunits/ceph-helpers.sh:988: wait_for_clean: (( timer >= 120 )) .../qa/workunits/ceph-helpers.sh:993: wait_for_clean: get_num_active_clean .../qa/workunits/ceph-helpers.sh:863: get_num_active_clean: local 'expression=(' .../qa/workunits/ceph-helpers.sh:864: get_num_active_clean: expression+='contains(.,'\''active'\'') and ' .../qa/workunits/ceph-helpers.sh:865: get_num_active_clean: expression+='contains(.,'\''clean'\'') and ' .../qa/workunits/ceph-helpers.sh:866: get_num_active_clean: expression+='not(contains(.,'\''stale'\''))' .../qa/workunits/ceph-helpers.sh:867: get_num_active_clean: expression+=')' .../qa/workunits/ceph-helpers.sh:871: get_num_active_clean: ceph --format xml pg dump pgs .../qa/workunits/ceph-helpers.sh:872: get_num_active_clean: xmlstarlet sel -t -m '//pg_stat/state[(contains(.,'\''active'\'') and contains(.,'\''clean'\'') and not(contains(.,'\''stale'\'')))]' -v . -n .../qa/workunits/ceph-helpers.sh:873: get_num_active_clean: grep -v '^$' .../qa/workunits/ceph-helpers.sh:873: get_num_active_clean: wc -l ../qa/workunits/ceph-helpers.sh:993: wait_for_clean: cur_active_clean=0 ../qa/workunits/ceph-helpers.sh:994: wait_for_clean: test 0 '!=' 0 ../qa/workunits/ceph-helpers.sh:998: wait_for_clean: sleep 1 .../qa/workunits/ceph-helpers.sh:999: wait_for_clean: expr 32 + 1 ../qa/workunits/ceph-helpers.sh:999: wait_for_clean: timer=33 ../qa/workunits/ceph-helpers.sh:985: wait_for_clean: is_clean .../qa/workunits/ceph-helpers.sh:952: is_clean: get_num_pgs .../qa/workunits/ceph-helpers.sh:898: get_num_pgs: ceph --format xml status .../qa/workunits/ceph-helpers.sh:899: get_num_pgs: xmlstarlet sel -t -m //pgmap/num_pgs -v . ../qa/workunits/ceph-helpers.sh:952: is_clean: num_pgs=4 ../qa/workunits/ceph-helpers.sh:953: is_clean: test 4 '!=' 0 .../qa/workunits/ceph-helpers.sh:954: is_clean: get_num_active_clean .../qa/workunits/ceph-helpers.sh:863: get_num_active_clean: local 'expression=(' .../qa/workunits/ceph-helpers.sh:864: get_num_active_clean: expression+='contains(.,'\''active'\'') and ' .../qa/workunits/ceph-helpers.sh:865: get_num_active_clean: expression+='contains(.,'\''clean'\'') and ' .../qa/workunits/ceph-helpers.sh:866: get_num_active_clean: expression+='not(contains(.,'\''stale'\''))' .../qa/workunits/ceph-helpers.sh:867: get_num_active_clean: expression+=')' .../qa/workunits/ceph-helpers.sh:871: get_num_active_clean: ceph --format xml pg dump pgs .../qa/workunits/ceph-helpers.sh:872: get_num_active_clean: xmlstarlet sel -t -m '//pg_stat/state[(contains(.,'\''active'\'') and contains(.,'\''clean'\'') and not(contains(.,'\''stale'\'')))]' -v . -n .../qa/workunits/ceph-helpers.sh:873: get_num_active_clean: grep -v '^$' .../qa/workunits/ceph-helpers.sh:873: get_num_active_clean: wc -l ../qa/workunits/ceph-helpers.sh:954: is_clean: test 0 = 4 ../qa/workunits/ceph-helpers.sh:954: is_clean: return 1 ../qa/workunits/ceph-helpers.sh:986: wait_for_clean: get_is_making_recovery_progress .../qa/workunits/ceph-helpers.sh:839: get_is_making_recovery_progress: ceph --format xml status .../qa/workunits/ceph-helpers.sh:839: get_is_making_recovery_progress: xmlstarlet sel -t -m //pgmap/recovering_keys_per_sec -v . -o ' ' -t -m //pgmap/recovering_bytes_per_sec -v . -o ' ' -t -m //pgmap/recovering_objects_per_sec -v . ../qa/workunits/ceph-helpers.sh:839: get_is_making_recovery_progress: local progress= ../qa/workunits/ceph-helpers.sh:840: get_is_making_recovery_progress: test -n '' ../qa/workunits/ceph-helpers.sh:988: wait_for_clean: (( timer >= 120 )) .../qa/workunits/ceph-helpers.sh:993: wait_for_clean: get_num_active_clean .../qa/workunits/ceph-helpers.sh:863: get_num_active_clean: local 'expression=(' .../qa/workunits/ceph-helpers.sh:864: get_num_active_clean: expression+='contains(.,'\''active'\'') and ' .../qa/workunits/ceph-helpers.sh:865: get_num_active_clean: expression+='contains(.,'\''clean'\'') and ' .../qa/workunits/ceph-helpers.sh:866: get_num_active_clean: expression+='not(contains(.,'\''stale'\''))' .../qa/workunits/ceph-helpers.sh:867: get_num_active_clean: expression+=')' .../qa/workunits/ceph-helpers.sh:871: get_num_active_clean: ceph --format xml pg dump pgs .../qa/workunits/ceph-helpers.sh:872: get_num_active_clean: xmlstarlet sel -t -m '//pg_stat/state[(contains(.,'\''active'\'') and contains(.,'\''clean'\'') and not(contains(.,'\''stale'\'')))]' -v . -n .../qa/workunits/ceph-helpers.sh:873: get_num_active_clean: grep -v '^$' .../qa/workunits/ceph-helpers.sh:873: get_num_active_clean: wc -l ../qa/workunits/ceph-helpers.sh:993: wait_for_clean: cur_active_clean=0 ../qa/workunits/ceph-helpers.sh:994: wait_for_clean: test 0 '!=' 0 ../qa/workunits/ceph-helpers.sh:998: wait_for_clean: sleep 1 .../qa/workunits/ceph-helpers.sh:999: wait_for_clean: expr 33 + 1 ../qa/workunits/ceph-helpers.sh:999: wait_for_clean: timer=34 ../qa/workunits/ceph-helpers.sh:985: wait_for_clean: is_clean .../qa/workunits/ceph-helpers.sh:952: is_clean: get_num_pgs .../qa/workunits/ceph-helpers.sh:898: get_num_pgs: ceph --format xml status .../qa/workunits/ceph-helpers.sh:899: get_num_pgs: xmlstarlet sel -t -m //pgmap/num_pgs -v . ../qa/workunits/ceph-helpers.sh:952: is_clean: num_pgs=4 ../qa/workunits/ceph-helpers.sh:953: is_clean: test 4 '!=' 0 .../qa/workunits/ceph-helpers.sh:954: is_clean: get_num_active_clean .../qa/workunits/ceph-helpers.sh:863: get_num_active_clean: local 'expression=(' .../qa/workunits/ceph-helpers.sh:864: get_num_active_clean: expression+='contains(.,'\''active'\'') and ' .../qa/workunits/ceph-helpers.sh:865: get_num_active_clean: expression+='contains(.,'\''clean'\'') and ' .../qa/workunits/ceph-helpers.sh:866: get_num_active_clean: expression+='not(contains(.,'\''stale'\''))' .../qa/workunits/ceph-helpers.sh:867: get_num_active_clean: expression+=')' .../qa/workunits/ceph-helpers.sh:871: get_num_active_clean: ceph --format xml pg dump pgs .../qa/workunits/ceph-helpers.sh:872: get_num_active_clean: xmlstarlet sel -t -m '//pg_stat/state[(contains(.,'\''active'\'') and contains(.,'\''clean'\'') and not(contains(.,'\''stale'\'')))]' -v . -n .../qa/workunits/ceph-helpers.sh:873: get_num_active_clean: grep -v '^$' .../qa/workunits/ceph-helpers.sh:873: get_num_active_clean: wc -l ../qa/workunits/ceph-helpers.sh:954: is_clean: test 0 = 4 ../qa/workunits/ceph-helpers.sh:954: is_clean: return 1 ../qa/workunits/ceph-helpers.sh:986: wait_for_clean: get_is_making_recovery_progress .../qa/workunits/ceph-helpers.sh:839: get_is_making_recovery_progress: ceph --format xml status .../qa/workunits/ceph-helpers.sh:839: get_is_making_recovery_progress: xmlstarlet sel -t -m //pgmap/recovering_keys_per_sec -v . -o ' ' -t -m //pgmap/recovering_bytes_per_sec -v . -o ' ' -t -m //pgmap/recovering_objects_per_sec -v . ../qa/workunits/ceph-helpers.sh:839: get_is_making_recovery_progress: local progress= ../qa/workunits/ceph-helpers.sh:840: get_is_making_recovery_progress: test -n '' ../qa/workunits/ceph-helpers.sh:988: wait_for_clean: (( timer >= 120 )) .../qa/workunits/ceph-helpers.sh:993: wait_for_clean: get_num_active_clean .../qa/workunits/ceph-helpers.sh:863: get_num_active_clean: local 'expression=(' .../qa/workunits/ceph-helpers.sh:864: get_num_active_clean: expression+='contains(.,'\''active'\'') and ' .../qa/workunits/ceph-helpers.sh:865: get_num_active_clean: expression+='contains(.,'\''clean'\'') and ' .../qa/workunits/ceph-helpers.sh:866: get_num_active_clean: expression+='not(contains(.,'\''stale'\''))' .../qa/workunits/ceph-helpers.sh:867: get_num_active_clean: expression+=')' .../qa/workunits/ceph-helpers.sh:871: get_num_active_clean: ceph --format xml pg dump pgs .../qa/workunits/ceph-helpers.sh:872: get_num_active_clean: xmlstarlet sel -t -m '//pg_stat/state[(contains(.,'\''active'\'') and contains(.,'\''clean'\'') and not(contains(.,'\''stale'\'')))]' -v . -n .../qa/workunits/ceph-helpers.sh:873: get_num_active_clean: grep -v '^$' .../qa/workunits/ceph-helpers.sh:873: get_num_active_clean: wc -l ../qa/workunits/ceph-helpers.sh:993: wait_for_clean: cur_active_clean=0 ../qa/workunits/ceph-helpers.sh:994: wait_for_clean: test 0 '!=' 0 ../qa/workunits/ceph-helpers.sh:998: wait_for_clean: sleep 1 .../qa/workunits/ceph-helpers.sh:999: wait_for_clean: expr 34 + 1 ../qa/workunits/ceph-helpers.sh:999: wait_for_clean: timer=35 ../qa/workunits/ceph-helpers.sh:985: wait_for_clean: is_clean .../qa/workunits/ceph-helpers.sh:952: is_clean: get_num_pgs .../qa/workunits/ceph-helpers.sh:898: get_num_pgs: ceph --format xml status .../qa/workunits/ceph-helpers.sh:899: get_num_pgs: xmlstarlet sel -t -m //pgmap/num_pgs -v . ../qa/workunits/ceph-helpers.sh:952: is_clean: num_pgs=4 ../qa/workunits/ceph-helpers.sh:953: is_clean: test 4 '!=' 0 .../qa/workunits/ceph-helpers.sh:954: is_clean: get_num_active_clean .../qa/workunits/ceph-helpers.sh:863: get_num_active_clean: local 'expression=(' .../qa/workunits/ceph-helpers.sh:864: get_num_active_clean: expression+='contains(.,'\''active'\'') and ' .../qa/workunits/ceph-helpers.sh:865: get_num_active_clean: expression+='contains(.,'\''clean'\'') and ' .../qa/workunits/ceph-helpers.sh:866: get_num_active_clean: expression+='not(contains(.,'\''stale'\''))' .../qa/workunits/ceph-helpers.sh:867: get_num_active_clean: expression+=')' .../qa/workunits/ceph-helpers.sh:871: get_num_active_clean: ceph --format xml pg dump pgs .../qa/workunits/ceph-helpers.sh:872: get_num_active_clean: xmlstarlet sel -t -m '//pg_stat/state[(contains(.,'\''active'\'') and contains(.,'\''clean'\'') and not(contains(.,'\''stale'\'')))]' -v . -n .../qa/workunits/ceph-helpers.sh:873: get_num_active_clean: wc -l .../qa/workunits/ceph-helpers.sh:873: get_num_active_clean: grep -v '^$' ../qa/workunits/ceph-helpers.sh:954: is_clean: test 0 = 4 ../qa/workunits/ceph-helpers.sh:954: is_clean: return 1 ../qa/workunits/ceph-helpers.sh:986: wait_for_clean: get_is_making_recovery_progress .../qa/workunits/ceph-helpers.sh:839: get_is_making_recovery_progress: ceph --format xml status .../qa/workunits/ceph-helpers.sh:839: get_is_making_recovery_progress: xmlstarlet sel -t -m //pgmap/recovering_keys_per_sec -v . -o ' ' -t -m //pgmap/recovering_bytes_per_sec -v . -o ' ' -t -m //pgmap/recovering_objects_per_sec -v . ../qa/workunits/ceph-helpers.sh:839: get_is_making_recovery_progress: local progress= ../qa/workunits/ceph-helpers.sh:840: get_is_making_recovery_progress: test -n '' ../qa/workunits/ceph-helpers.sh:988: wait_for_clean: (( timer >= 120 )) .../qa/workunits/ceph-helpers.sh:993: wait_for_clean: get_num_active_clean .../qa/workunits/ceph-helpers.sh:863: get_num_active_clean: local 'expression=(' .../qa/workunits/ceph-helpers.sh:864: get_num_active_clean: expression+='contains(.,'\''active'\'') and ' .../qa/workunits/ceph-helpers.sh:865: get_num_active_clean: expression+='contains(.,'\''clean'\'') and ' .../qa/workunits/ceph-helpers.sh:866: get_num_active_clean: expression+='not(contains(.,'\''stale'\''))' .../qa/workunits/ceph-helpers.sh:867: get_num_active_clean: expression+=')' .../qa/workunits/ceph-helpers.sh:871: get_num_active_clean: ceph --format xml pg dump pgs .../qa/workunits/ceph-helpers.sh:872: get_num_active_clean: xmlstarlet sel -t -m '//pg_stat/state[(contains(.,'\''active'\'') and contains(.,'\''clean'\'') and not(contains(.,'\''stale'\'')))]' -v . -n .../qa/workunits/ceph-helpers.sh:873: get_num_active_clean: grep -v '^$' .../qa/workunits/ceph-helpers.sh:873: get_num_active_clean: wc -l ../qa/workunits/ceph-helpers.sh:993: wait_for_clean: cur_active_clean=0 ../qa/workunits/ceph-helpers.sh:994: wait_for_clean: test 0 '!=' 0 ../qa/workunits/ceph-helpers.sh:998: wait_for_clean: sleep 1 .../qa/workunits/ceph-helpers.sh:999: wait_for_clean: expr 35 + 1 ../qa/workunits/ceph-helpers.sh:999: wait_for_clean: timer=36 ../qa/workunits/ceph-helpers.sh:985: wait_for_clean: is_clean .../qa/workunits/ceph-helpers.sh:952: is_clean: get_num_pgs .../qa/workunits/ceph-helpers.sh:898: get_num_pgs: ceph --format xml status .../qa/workunits/ceph-helpers.sh:899: get_num_pgs: xmlstarlet sel -t -m //pgmap/num_pgs -v . ../qa/workunits/ceph-helpers.sh:952: is_clean: num_pgs=4 ../qa/workunits/ceph-helpers.sh:953: is_clean: test 4 '!=' 0 .../qa/workunits/ceph-helpers.sh:954: is_clean: get_num_active_clean .../qa/workunits/ceph-helpers.sh:863: get_num_active_clean: local 'expression=(' .../qa/workunits/ceph-helpers.sh:864: get_num_active_clean: expression+='contains(.,'\''active'\'') and ' .../qa/workunits/ceph-helpers.sh:865: get_num_active_clean: expression+='contains(.,'\''clean'\'') and ' .../qa/workunits/ceph-helpers.sh:866: get_num_active_clean: expression+='not(contains(.,'\''stale'\''))' .../qa/workunits/ceph-helpers.sh:867: get_num_active_clean: expression+=')' .../qa/workunits/ceph-helpers.sh:871: get_num_active_clean: ceph --format xml pg dump pgs .../qa/workunits/ceph-helpers.sh:872: get_num_active_clean: xmlstarlet sel -t -m '//pg_stat/state[(contains(.,'\''active'\'') and contains(.,'\''clean'\'') and not(contains(.,'\''stale'\'')))]' -v . -n .../qa/workunits/ceph-helpers.sh:873: get_num_active_clean: grep -v '^$' .../qa/workunits/ceph-helpers.sh:873: get_num_active_clean: wc -l ../qa/workunits/ceph-helpers.sh:954: is_clean: test 0 = 4 ../qa/workunits/ceph-helpers.sh:954: is_clean: return 1 ../qa/workunits/ceph-helpers.sh:986: wait_for_clean: get_is_making_recovery_progress .../qa/workunits/ceph-helpers.sh:839: get_is_making_recovery_progress: ceph --format xml status .../qa/workunits/ceph-helpers.sh:839: get_is_making_recovery_progress: xmlstarlet sel -t -m //pgmap/recovering_keys_per_sec -v . -o ' ' -t -m //pgmap/recovering_bytes_per_sec -v . -o ' ' -t -m //pgmap/recovering_objects_per_sec -v . ../qa/workunits/ceph-helpers.sh:839: get_is_making_recovery_progress: local progress= ../qa/workunits/ceph-helpers.sh:840: get_is_making_recovery_progress: test -n '' ../qa/workunits/ceph-helpers.sh:988: wait_for_clean: (( timer >= 120 )) .../qa/workunits/ceph-helpers.sh:993: wait_for_clean: get_num_active_clean .../qa/workunits/ceph-helpers.sh:863: get_num_active_clean: local 'expression=(' .../qa/workunits/ceph-helpers.sh:864: get_num_active_clean: expression+='contains(.,'\''active'\'') and ' .../qa/workunits/ceph-helpers.sh:865: get_num_active_clean: expression+='contains(.,'\''clean'\'') and ' .../qa/workunits/ceph-helpers.sh:866: get_num_active_clean: expression+='not(contains(.,'\''stale'\''))' .../qa/workunits/ceph-helpers.sh:867: get_num_active_clean: expression+=')' .../qa/workunits/ceph-helpers.sh:871: get_num_active_clean: ceph --format xml pg dump pgs .../qa/workunits/ceph-helpers.sh:872: get_num_active_clean: xmlstarlet sel -t -m '//pg_stat/state[(contains(.,'\''active'\'') and contains(.,'\''clean'\'') and not(contains(.,'\''stale'\'')))]' -v . -n .../qa/workunits/ceph-helpers.sh:873: get_num_active_clean: grep -v '^$' .../qa/workunits/ceph-helpers.sh:873: get_num_active_clean: wc -l ../qa/workunits/ceph-helpers.sh:993: wait_for_clean: cur_active_clean=0 ../qa/workunits/ceph-helpers.sh:994: wait_for_clean: test 0 '!=' 0 ../qa/workunits/ceph-helpers.sh:998: wait_for_clean: sleep 1 .../qa/workunits/ceph-helpers.sh:999: wait_for_clean: expr 36 + 1 ../qa/workunits/ceph-helpers.sh:999: wait_for_clean: timer=37 ../qa/workunits/ceph-helpers.sh:985: wait_for_clean: is_clean .../qa/workunits/ceph-helpers.sh:952: is_clean: get_num_pgs .../qa/workunits/ceph-helpers.sh:898: get_num_pgs: ceph --format xml status .../qa/workunits/ceph-helpers.sh:899: get_num_pgs: xmlstarlet sel -t -m //pgmap/num_pgs -v . ../qa/workunits/ceph-helpers.sh:952: is_clean: num_pgs=4 ../qa/workunits/ceph-helpers.sh:953: is_clean: test 4 '!=' 0 .../qa/workunits/ceph-helpers.sh:954: is_clean: get_num_active_clean .../qa/workunits/ceph-helpers.sh:863: get_num_active_clean: local 'expression=(' .../qa/workunits/ceph-helpers.sh:864: get_num_active_clean: expression+='contains(.,'\''active'\'') and ' .../qa/workunits/ceph-helpers.sh:865: get_num_active_clean: expression+='contains(.,'\''clean'\'') and ' .../qa/workunits/ceph-helpers.sh:866: get_num_active_clean: expression+='not(contains(.,'\''stale'\''))' .../qa/workunits/ceph-helpers.sh:867: get_num_active_clean: expression+=')' .../qa/workunits/ceph-helpers.sh:871: get_num_active_clean: ceph --format xml pg dump pgs .../qa/workunits/ceph-helpers.sh:872: get_num_active_clean: xmlstarlet sel -t -m '//pg_stat/state[(contains(.,'\''active'\'') and contains(.,'\''clean'\'') and not(contains(.,'\''stale'\'')))]' -v . -n .../qa/workunits/ceph-helpers.sh:873: get_num_active_clean: grep -v '^$' .../qa/workunits/ceph-helpers.sh:873: get_num_active_clean: wc -l ../qa/workunits/ceph-helpers.sh:954: is_clean: test 0 = 4 ../qa/workunits/ceph-helpers.sh:954: is_clean: return 1 ../qa/workunits/ceph-helpers.sh:986: wait_for_clean: get_is_making_recovery_progress .../qa/workunits/ceph-helpers.sh:839: get_is_making_recovery_progress: ceph --format xml status .../qa/workunits/ceph-helpers.sh:839: get_is_making_recovery_progress: xmlstarlet sel -t -m //pgmap/recovering_keys_per_sec -v . -o ' ' -t -m //pgmap/recovering_bytes_per_sec -v . -o ' ' -t -m //pgmap/recovering_objects_per_sec -v . ../qa/workunits/ceph-helpers.sh:839: get_is_making_recovery_progress: local progress= ../qa/workunits/ceph-helpers.sh:840: get_is_making_recovery_progress: test -n '' ../qa/workunits/ceph-helpers.sh:988: wait_for_clean: (( timer >= 120 )) .../qa/workunits/ceph-helpers.sh:993: wait_for_clean: get_num_active_clean .../qa/workunits/ceph-helpers.sh:863: get_num_active_clean: local 'expression=(' .../qa/workunits/ceph-helpers.sh:864: get_num_active_clean: expression+='contains(.,'\''active'\'') and ' .../qa/workunits/ceph-helpers.sh:865: get_num_active_clean: expression+='contains(.,'\''clean'\'') and ' .../qa/workunits/ceph-helpers.sh:866: get_num_active_clean: expression+='not(contains(.,'\''stale'\''))' .../qa/workunits/ceph-helpers.sh:867: get_num_active_clean: expression+=')' .../qa/workunits/ceph-helpers.sh:871: get_num_active_clean: ceph --format xml pg dump pgs .../qa/workunits/ceph-helpers.sh:872: get_num_active_clean: xmlstarlet sel -t -m '//pg_stat/state[(contains(.,'\''active'\'') and contains(.,'\''clean'\'') and not(contains(.,'\''stale'\'')))]' -v . -n .../qa/workunits/ceph-helpers.sh:873: get_num_active_clean: grep -v '^$' .../qa/workunits/ceph-helpers.sh:873: get_num_active_clean: wc -l ../qa/workunits/ceph-helpers.sh:993: wait_for_clean: cur_active_clean=0 ../qa/workunits/ceph-helpers.sh:994: wait_for_clean: test 0 '!=' 0 ../qa/workunits/ceph-helpers.sh:998: wait_for_clean: sleep 1 .../qa/workunits/ceph-helpers.sh:999: wait_for_clean: expr 37 + 1 ../qa/workunits/ceph-helpers.sh:999: wait_for_clean: timer=38 ../qa/workunits/ceph-helpers.sh:985: wait_for_clean: is_clean .../qa/workunits/ceph-helpers.sh:952: is_clean: get_num_pgs .../qa/workunits/ceph-helpers.sh:898: get_num_pgs: ceph --format xml status .../qa/workunits/ceph-helpers.sh:899: get_num_pgs: xmlstarlet sel -t -m //pgmap/num_pgs -v . ../qa/workunits/ceph-helpers.sh:952: is_clean: num_pgs=4 ../qa/workunits/ceph-helpers.sh:953: is_clean: test 4 '!=' 0 .../qa/workunits/ceph-helpers.sh:954: is_clean: get_num_active_clean .../qa/workunits/ceph-helpers.sh:863: get_num_active_clean: local 'expression=(' .../qa/workunits/ceph-helpers.sh:864: get_num_active_clean: expression+='contains(.,'\''active'\'') and ' .../qa/workunits/ceph-helpers.sh:865: get_num_active_clean: expression+='contains(.,'\''clean'\'') and ' .../qa/workunits/ceph-helpers.sh:866: get_num_active_clean: expression+='not(contains(.,'\''stale'\''))' .../qa/workunits/ceph-helpers.sh:867: get_num_active_clean: expression+=')' .../qa/workunits/ceph-helpers.sh:871: get_num_active_clean: ceph --format xml pg dump pgs .../qa/workunits/ceph-helpers.sh:872: get_num_active_clean: xmlstarlet sel -t -m '//pg_stat/state[(contains(.,'\''active'\'') and contains(.,'\''clean'\'') and not(contains(.,'\''stale'\'')))]' -v . -n .../qa/workunits/ceph-helpers.sh:873: get_num_active_clean: grep -v '^$' .../qa/workunits/ceph-helpers.sh:873: get_num_active_clean: wc -l ../qa/workunits/ceph-helpers.sh:954: is_clean: test 0 = 4 ../qa/workunits/ceph-helpers.sh:954: is_clean: return 1 ../qa/workunits/ceph-helpers.sh:986: wait_for_clean: get_is_making_recovery_progress .../qa/workunits/ceph-helpers.sh:839: get_is_making_recovery_progress: ceph --format xml status .../qa/workunits/ceph-helpers.sh:839: get_is_making_recovery_progress: xmlstarlet sel -t -m //pgmap/recovering_keys_per_sec -v . -o ' ' -t -m //pgmap/recovering_bytes_per_sec -v . -o ' ' -t -m //pgmap/recovering_objects_per_sec -v . ../qa/workunits/ceph-helpers.sh:839: get_is_making_recovery_progress: local progress= ../qa/workunits/ceph-helpers.sh:840: get_is_making_recovery_progress: test -n '' ../qa/workunits/ceph-helpers.sh:988: wait_for_clean: (( timer >= 120 )) .../qa/workunits/ceph-helpers.sh:993: wait_for_clean: get_num_active_clean .../qa/workunits/ceph-helpers.sh:863: get_num_active_clean: local 'expression=(' .../qa/workunits/ceph-helpers.sh:864: get_num_active_clean: expression+='contains(.,'\''active'\'') and ' .../qa/workunits/ceph-helpers.sh:865: get_num_active_clean: expression+='contains(.,'\''clean'\'') and ' .../qa/workunits/ceph-helpers.sh:866: get_num_active_clean: expression+='not(contains(.,'\''stale'\''))' .../qa/workunits/ceph-helpers.sh:867: get_num_active_clean: expression+=')' .../qa/workunits/ceph-helpers.sh:871: get_num_active_clean: ceph --format xml pg dump pgs .../qa/workunits/ceph-helpers.sh:872: get_num_active_clean: xmlstarlet sel -t -m '//pg_stat/state[(contains(.,'\''active'\'') and contains(.,'\''clean'\'') and not(contains(.,'\''stale'\'')))]' -v . -n .../qa/workunits/ceph-helpers.sh:873: get_num_active_clean: grep -v '^$' .../qa/workunits/ceph-helpers.sh:873: get_num_active_clean: wc -l ../qa/workunits/ceph-helpers.sh:993: wait_for_clean: cur_active_clean=0 ../qa/workunits/ceph-helpers.sh:994: wait_for_clean: test 0 '!=' 0 ../qa/workunits/ceph-helpers.sh:998: wait_for_clean: sleep 1 .../qa/workunits/ceph-helpers.sh:999: wait_for_clean: expr 38 + 1 ../qa/workunits/ceph-helpers.sh:999: wait_for_clean: timer=39 ../qa/workunits/ceph-helpers.sh:985: wait_for_clean: is_clean .../qa/workunits/ceph-helpers.sh:952: is_clean: get_num_pgs .../qa/workunits/ceph-helpers.sh:898: get_num_pgs: ceph --format xml status .../qa/workunits/ceph-helpers.sh:899: get_num_pgs: xmlstarlet sel -t -m //pgmap/num_pgs -v . ../qa/workunits/ceph-helpers.sh:952: is_clean: num_pgs=4 ../qa/workunits/ceph-helpers.sh:953: is_clean: test 4 '!=' 0 .../qa/workunits/ceph-helpers.sh:954: is_clean: get_num_active_clean .../qa/workunits/ceph-helpers.sh:863: get_num_active_clean: local 'expression=(' .../qa/workunits/ceph-helpers.sh:864: get_num_active_clean: expression+='contains(.,'\''active'\'') and ' .../qa/workunits/ceph-helpers.sh:865: get_num_active_clean: expression+='contains(.,'\''clean'\'') and ' .../qa/workunits/ceph-helpers.sh:866: get_num_active_clean: expression+='not(contains(.,'\''stale'\''))' .../qa/workunits/ceph-helpers.sh:867: get_num_active_clean: expression+=')' .../qa/workunits/ceph-helpers.sh:871: get_num_active_clean: ceph --format xml pg dump pgs .../qa/workunits/ceph-helpers.sh:872: get_num_active_clean: xmlstarlet sel -t -m '//pg_stat/state[(contains(.,'\''active'\'') and contains(.,'\''clean'\'') and not(contains(.,'\''stale'\'')))]' -v . -n .../qa/workunits/ceph-helpers.sh:873: get_num_active_clean: wc -l .../qa/workunits/ceph-helpers.sh:873: get_num_active_clean: grep -v '^$' ../qa/workunits/ceph-helpers.sh:954: is_clean: test 0 = 4 ../qa/workunits/ceph-helpers.sh:954: is_clean: return 1 ../qa/workunits/ceph-helpers.sh:986: wait_for_clean: get_is_making_recovery_progress .../qa/workunits/ceph-helpers.sh:839: get_is_making_recovery_progress: ceph --format xml status .../qa/workunits/ceph-helpers.sh:839: get_is_making_recovery_progress: xmlstarlet sel -t -m //pgmap/recovering_keys_per_sec -v . -o ' ' -t -m //pgmap/recovering_bytes_per_sec -v . -o ' ' -t -m //pgmap/recovering_objects_per_sec -v . ../qa/workunits/ceph-helpers.sh:839: get_is_making_recovery_progress: local progress= ../qa/workunits/ceph-helpers.sh:840: get_is_making_recovery_progress: test -n '' ../qa/workunits/ceph-helpers.sh:988: wait_for_clean: (( timer >= 120 )) .../qa/workunits/ceph-helpers.sh:993: wait_for_clean: get_num_active_clean .../qa/workunits/ceph-helpers.sh:863: get_num_active_clean: local 'expression=(' .../qa/workunits/ceph-helpers.sh:864: get_num_active_clean: expression+='contains(.,'\''active'\'') and ' .../qa/workunits/ceph-helpers.sh:865: get_num_active_clean: expression+='contains(.,'\''clean'\'') and ' .../qa/workunits/ceph-helpers.sh:866: get_num_active_clean: expression+='not(contains(.,'\''stale'\''))' .../qa/workunits/ceph-helpers.sh:867: get_num_active_clean: expression+=')' .../qa/workunits/ceph-helpers.sh:871: get_num_active_clean: ceph --format xml pg dump pgs .../qa/workunits/ceph-helpers.sh:872: get_num_active_clean: xmlstarlet sel -t -m '//pg_stat/state[(contains(.,'\''active'\'') and contains(.,'\''clean'\'') and not(contains(.,'\''stale'\'')))]' -v . -n .../qa/workunits/ceph-helpers.sh:873: get_num_active_clean: wc -l .../qa/workunits/ceph-helpers.sh:873: get_num_active_clean: grep -v '^$' ../qa/workunits/ceph-helpers.sh:993: wait_for_clean: cur_active_clean=0 ../qa/workunits/ceph-helpers.sh:994: wait_for_clean: test 0 '!=' 0 ../qa/workunits/ceph-helpers.sh:998: wait_for_clean: sleep 1 .../qa/workunits/ceph-helpers.sh:999: wait_for_clean: expr 39 + 1 ../qa/workunits/ceph-helpers.sh:999: wait_for_clean: timer=40 ../qa/workunits/ceph-helpers.sh:985: wait_for_clean: is_clean .../qa/workunits/ceph-helpers.sh:952: is_clean: get_num_pgs .../qa/workunits/ceph-helpers.sh:898: get_num_pgs: ceph --format xml status .../qa/workunits/ceph-helpers.sh:899: get_num_pgs: xmlstarlet sel -t -m //pgmap/num_pgs -v . ../qa/workunits/ceph-helpers.sh:952: is_clean: num_pgs=4 ../qa/workunits/ceph-helpers.sh:953: is_clean: test 4 '!=' 0 .../qa/workunits/ceph-helpers.sh:954: is_clean: get_num_active_clean .../qa/workunits/ceph-helpers.sh:863: get_num_active_clean: local 'expression=(' .../qa/workunits/ceph-helpers.sh:864: get_num_active_clean: expression+='contains(.,'\''active'\'') and ' .../qa/workunits/ceph-helpers.sh:865: get_num_active_clean: expression+='contains(.,'\''clean'\'') and ' .../qa/workunits/ceph-helpers.sh:866: get_num_active_clean: expression+='not(contains(.,'\''stale'\''))' .../qa/workunits/ceph-helpers.sh:867: get_num_active_clean: expression+=')' .../qa/workunits/ceph-helpers.sh:871: get_num_active_clean: ceph --format xml pg dump pgs .../qa/workunits/ceph-helpers.sh:872: get_num_active_clean: xmlstarlet sel -t -m '//pg_stat/state[(contains(.,'\''active'\'') and contains(.,'\''clean'\'') and not(contains(.,'\''stale'\'')))]' -v . -n .../qa/workunits/ceph-helpers.sh:873: get_num_active_clean: grep -v '^$' .../qa/workunits/ceph-helpers.sh:873: get_num_active_clean: wc -l ../qa/workunits/ceph-helpers.sh:954: is_clean: test 0 = 4 ../qa/workunits/ceph-helpers.sh:954: is_clean: return 1 ../qa/workunits/ceph-helpers.sh:986: wait_for_clean: get_is_making_recovery_progress .../qa/workunits/ceph-helpers.sh:839: get_is_making_recovery_progress: ceph --format xml status .../qa/workunits/ceph-helpers.sh:839: get_is_making_recovery_progress: xmlstarlet sel -t -m //pgmap/recovering_keys_per_sec -v . -o ' ' -t -m //pgmap/recovering_bytes_per_sec -v . -o ' ' -t -m //pgmap/recovering_objects_per_sec -v . ../qa/workunits/ceph-helpers.sh:839: get_is_making_recovery_progress: local progress= ../qa/workunits/ceph-helpers.sh:840: get_is_making_recovery_progress: test -n '' ../qa/workunits/ceph-helpers.sh:988: wait_for_clean: (( timer >= 120 )) .../qa/workunits/ceph-helpers.sh:993: wait_for_clean: get_num_active_clean .../qa/workunits/ceph-helpers.sh:863: get_num_active_clean: local 'expression=(' .../qa/workunits/ceph-helpers.sh:864: get_num_active_clean: expression+='contains(.,'\''active'\'') and ' .../qa/workunits/ceph-helpers.sh:865: get_num_active_clean: expression+='contains(.,'\''clean'\'') and ' .../qa/workunits/ceph-helpers.sh:866: get_num_active_clean: expression+='not(contains(.,'\''stale'\''))' .../qa/workunits/ceph-helpers.sh:867: get_num_active_clean: expression+=')' .../qa/workunits/ceph-helpers.sh:871: get_num_active_clean: ceph --format xml pg dump pgs .../qa/workunits/ceph-helpers.sh:872: get_num_active_clean: xmlstarlet sel -t -m '//pg_stat/state[(contains(.,'\''active'\'') and contains(.,'\''clean'\'') and not(contains(.,'\''stale'\'')))]' -v . -n .../qa/workunits/ceph-helpers.sh:873: get_num_active_clean: grep -v '^$' .../qa/workunits/ceph-helpers.sh:873: get_num_active_clean: wc -l ../qa/workunits/ceph-helpers.sh:993: wait_for_clean: cur_active_clean=0 ../qa/workunits/ceph-helpers.sh:994: wait_for_clean: test 0 '!=' 0 ../qa/workunits/ceph-helpers.sh:998: wait_for_clean: sleep 1 .../qa/workunits/ceph-helpers.sh:999: wait_for_clean: expr 40 + 1 ../qa/workunits/ceph-helpers.sh:999: wait_for_clean: timer=41 ../qa/workunits/ceph-helpers.sh:985: wait_for_clean: is_clean .../qa/workunits/ceph-helpers.sh:952: is_clean: get_num_pgs .../qa/workunits/ceph-helpers.sh:898: get_num_pgs: ceph --format xml status .../qa/workunits/ceph-helpers.sh:899: get_num_pgs: xmlstarlet sel -t -m //pgmap/num_pgs -v . ../qa/workunits/ceph-helpers.sh:952: is_clean: num_pgs=4 ../qa/workunits/ceph-helpers.sh:953: is_clean: test 4 '!=' 0 .../qa/workunits/ceph-helpers.sh:954: is_clean: get_num_active_clean .../qa/workunits/ceph-helpers.sh:863: get_num_active_clean: local 'expression=(' .../qa/workunits/ceph-helpers.sh:864: get_num_active_clean: expression+='contains(.,'\''active'\'') and ' .../qa/workunits/ceph-helpers.sh:865: get_num_active_clean: expression+='contains(.,'\''clean'\'') and ' .../qa/workunits/ceph-helpers.sh:866: get_num_active_clean: expression+='not(contains(.,'\''stale'\''))' .../qa/workunits/ceph-helpers.sh:867: get_num_active_clean: expression+=')' .../qa/workunits/ceph-helpers.sh:871: get_num_active_clean: ceph --format xml pg dump pgs .../qa/workunits/ceph-helpers.sh:873: get_num_active_clean: grep -v '^$' .../qa/workunits/ceph-helpers.sh:873: get_num_active_clean: wc -l .../qa/workunits/ceph-helpers.sh:872: get_num_active_clean: xmlstarlet sel -t -m '//pg_stat/state[(contains(.,'\''active'\'') and contains(.,'\''clean'\'') and not(contains(.,'\''stale'\'')))]' -v . -n ../qa/workunits/ceph-helpers.sh:954: is_clean: test 0 = 4 ../qa/workunits/ceph-helpers.sh:954: is_clean: return 1 ../qa/workunits/ceph-helpers.sh:986: wait_for_clean: get_is_making_recovery_progress .../qa/workunits/ceph-helpers.sh:839: get_is_making_recovery_progress: ceph --format xml status .../qa/workunits/ceph-helpers.sh:839: get_is_making_recovery_progress: xmlstarlet sel -t -m //pgmap/recovering_keys_per_sec -v . -o ' ' -t -m //pgmap/recovering_bytes_per_sec -v . -o ' ' -t -m //pgmap/recovering_objects_per_sec -v . ../qa/workunits/ceph-helpers.sh:839: get_is_making_recovery_progress: local progress= ../qa/workunits/ceph-helpers.sh:840: get_is_making_recovery_progress: test -n '' ../qa/workunits/ceph-helpers.sh:988: wait_for_clean: (( timer >= 120 )) .../qa/workunits/ceph-helpers.sh:993: wait_for_clean: get_num_active_clean .../qa/workunits/ceph-helpers.sh:863: get_num_active_clean: local 'expression=(' .../qa/workunits/ceph-helpers.sh:864: get_num_active_clean: expression+='contains(.,'\''active'\'') and ' .../qa/workunits/ceph-helpers.sh:865: get_num_active_clean: expression+='contains(.,'\''clean'\'') and ' .../qa/workunits/ceph-helpers.sh:866: get_num_active_clean: expression+='not(contains(.,'\''stale'\''))' .../qa/workunits/ceph-helpers.sh:867: get_num_active_clean: expression+=')' .../qa/workunits/ceph-helpers.sh:871: get_num_active_clean: ceph --format xml pg dump pgs .../qa/workunits/ceph-helpers.sh:873: get_num_active_clean: grep -v '^$' .../qa/workunits/ceph-helpers.sh:872: get_num_active_clean: xmlstarlet sel -t -m '//pg_stat/state[(contains(.,'\''active'\'') and contains(.,'\''clean'\'') and not(contains(.,'\''stale'\'')))]' -v . -n .../qa/workunits/ceph-helpers.sh:873: get_num_active_clean: wc -l ../qa/workunits/ceph-helpers.sh:993: wait_for_clean: cur_active_clean=0 ../qa/workunits/ceph-helpers.sh:994: wait_for_clean: test 0 '!=' 0 ../qa/workunits/ceph-helpers.sh:998: wait_for_clean: sleep 1 .../qa/workunits/ceph-helpers.sh:999: wait_for_clean: expr 41 + 1 ../qa/workunits/ceph-helpers.sh:999: wait_for_clean: timer=42 ../qa/workunits/ceph-helpers.sh:985: wait_for_clean: is_clean .../qa/workunits/ceph-helpers.sh:952: is_clean: get_num_pgs .../qa/workunits/ceph-helpers.sh:898: get_num_pgs: ceph --format xml status .../qa/workunits/ceph-helpers.sh:899: get_num_pgs: xmlstarlet sel -t -m //pgmap/num_pgs -v . ../qa/workunits/ceph-helpers.sh:952: is_clean: num_pgs=4 ../qa/workunits/ceph-helpers.sh:953: is_clean: test 4 '!=' 0 .../qa/workunits/ceph-helpers.sh:954: is_clean: get_num_active_clean .../qa/workunits/ceph-helpers.sh:863: get_num_active_clean: local 'expression=(' .../qa/workunits/ceph-helpers.sh:864: get_num_active_clean: expression+='contains(.,'\''active'\'') and ' .../qa/workunits/ceph-helpers.sh:865: get_num_active_clean: expression+='contains(.,'\''clean'\'') and ' .../qa/workunits/ceph-helpers.sh:866: get_num_active_clean: expression+='not(contains(.,'\''stale'\''))' .../qa/workunits/ceph-helpers.sh:867: get_num_active_clean: expression+=')' .../qa/workunits/ceph-helpers.sh:871: get_num_active_clean: ceph --format xml pg dump pgs .../qa/workunits/ceph-helpers.sh:872: get_num_active_clean: xmlstarlet sel -t -m '//pg_stat/state[(contains(.,'\''active'\'') and contains(.,'\''clean'\'') and not(contains(.,'\''stale'\'')))]' -v . -n .../qa/workunits/ceph-helpers.sh:873: get_num_active_clean: grep -v '^$' .../qa/workunits/ceph-helpers.sh:873: get_num_active_clean: wc -l ../qa/workunits/ceph-helpers.sh:954: is_clean: test 0 = 4 ../qa/workunits/ceph-helpers.sh:954: is_clean: return 1 ../qa/workunits/ceph-helpers.sh:986: wait_for_clean: get_is_making_recovery_progress .../qa/workunits/ceph-helpers.sh:839: get_is_making_recovery_progress: ceph --format xml status .../qa/workunits/ceph-helpers.sh:839: get_is_making_recovery_progress: xmlstarlet sel -t -m //pgmap/recovering_keys_per_sec -v . -o ' ' -t -m //pgmap/recovering_bytes_per_sec -v . -o ' ' -t -m //pgmap/recovering_objects_per_sec -v . ../qa/workunits/ceph-helpers.sh:839: get_is_making_recovery_progress: local progress= ../qa/workunits/ceph-helpers.sh:840: get_is_making_recovery_progress: test -n '' ../qa/workunits/ceph-helpers.sh:988: wait_for_clean: (( timer >= 120 )) .../qa/workunits/ceph-helpers.sh:993: wait_for_clean: get_num_active_clean .../qa/workunits/ceph-helpers.sh:863: get_num_active_clean: local 'expression=(' .../qa/workunits/ceph-helpers.sh:864: get_num_active_clean: expression+='contains(.,'\''active'\'') and ' .../qa/workunits/ceph-helpers.sh:865: get_num_active_clean: expression+='contains(.,'\''clean'\'') and ' .../qa/workunits/ceph-helpers.sh:866: get_num_active_clean: expression+='not(contains(.,'\''stale'\''))' .../qa/workunits/ceph-helpers.sh:867: get_num_active_clean: expression+=')' .../qa/workunits/ceph-helpers.sh:871: get_num_active_clean: ceph --format xml pg dump pgs .../qa/workunits/ceph-helpers.sh:872: get_num_active_clean: xmlstarlet sel -t -m '//pg_stat/state[(contains(.,'\''active'\'') and contains(.,'\''clean'\'') and not(contains(.,'\''stale'\'')))]' -v . -n .../qa/workunits/ceph-helpers.sh:873: get_num_active_clean: grep -v '^$' .../qa/workunits/ceph-helpers.sh:873: get_num_active_clean: wc -l ../qa/workunits/ceph-helpers.sh:993: wait_for_clean: cur_active_clean=0 ../qa/workunits/ceph-helpers.sh:994: wait_for_clean: test 0 '!=' 0 ../qa/workunits/ceph-helpers.sh:998: wait_for_clean: sleep 1 .../qa/workunits/ceph-helpers.sh:999: wait_for_clean: expr 42 + 1 ../qa/workunits/ceph-helpers.sh:999: wait_for_clean: timer=43 ../qa/workunits/ceph-helpers.sh:985: wait_for_clean: is_clean .../qa/workunits/ceph-helpers.sh:952: is_clean: get_num_pgs .../qa/workunits/ceph-helpers.sh:898: get_num_pgs: ceph --format xml status .../qa/workunits/ceph-helpers.sh:899: get_num_pgs: xmlstarlet sel -t -m //pgmap/num_pgs -v . ../qa/workunits/ceph-helpers.sh:952: is_clean: num_pgs=4 ../qa/workunits/ceph-helpers.sh:953: is_clean: test 4 '!=' 0 .../qa/workunits/ceph-helpers.sh:954: is_clean: get_num_active_clean .../qa/workunits/ceph-helpers.sh:863: get_num_active_clean: local 'expression=(' .../qa/workunits/ceph-helpers.sh:864: get_num_active_clean: expression+='contains(.,'\''active'\'') and ' .../qa/workunits/ceph-helpers.sh:865: get_num_active_clean: expression+='contains(.,'\''clean'\'') and ' .../qa/workunits/ceph-helpers.sh:866: get_num_active_clean: expression+='not(contains(.,'\''stale'\''))' .../qa/workunits/ceph-helpers.sh:867: get_num_active_clean: expression+=')' .../qa/workunits/ceph-helpers.sh:871: get_num_active_clean: ceph --format xml pg dump pgs .../qa/workunits/ceph-helpers.sh:872: get_num_active_clean: xmlstarlet sel -t -m '//pg_stat/state[(contains(.,'\''active'\'') and contains(.,'\''clean'\'') and not(contains(.,'\''stale'\'')))]' -v . -n .../qa/workunits/ceph-helpers.sh:873: get_num_active_clean: grep -v '^$' .../qa/workunits/ceph-helpers.sh:873: get_num_active_clean: wc -l ../qa/workunits/ceph-helpers.sh:954: is_clean: test 0 = 4 ../qa/workunits/ceph-helpers.sh:954: is_clean: return 1 ../qa/workunits/ceph-helpers.sh:986: wait_for_clean: get_is_making_recovery_progress .../qa/workunits/ceph-helpers.sh:839: get_is_making_recovery_progress: ceph --format xml status .../qa/workunits/ceph-helpers.sh:839: get_is_making_recovery_progress: xmlstarlet sel -t -m //pgmap/recovering_keys_per_sec -v . -o ' ' -t -m //pgmap/recovering_bytes_per_sec -v . -o ' ' -t -m //pgmap/recovering_objects_per_sec -v . ../qa/workunits/ceph-helpers.sh:839: get_is_making_recovery_progress: local progress= ../qa/workunits/ceph-helpers.sh:840: get_is_making_recovery_progress: test -n '' ../qa/workunits/ceph-helpers.sh:988: wait_for_clean: (( timer >= 120 )) .../qa/workunits/ceph-helpers.sh:993: wait_for_clean: get_num_active_clean .../qa/workunits/ceph-helpers.sh:863: get_num_active_clean: local 'expression=(' .../qa/workunits/ceph-helpers.sh:864: get_num_active_clean: expression+='contains(.,'\''active'\'') and ' .../qa/workunits/ceph-helpers.sh:865: get_num_active_clean: expression+='contains(.,'\''clean'\'') and ' .../qa/workunits/ceph-helpers.sh:866: get_num_active_clean: expression+='not(contains(.,'\''stale'\''))' .../qa/workunits/ceph-helpers.sh:867: get_num_active_clean: expression+=')' .../qa/workunits/ceph-helpers.sh:871: get_num_active_clean: ceph --format xml pg dump pgs .../qa/workunits/ceph-helpers.sh:872: get_num_active_clean: xmlstarlet sel -t -m '//pg_stat/state[(contains(.,'\''active'\'') and contains(.,'\''clean'\'') and not(contains(.,'\''stale'\'')))]' -v . -n .../qa/workunits/ceph-helpers.sh:873: get_num_active_clean: grep -v '^$' .../qa/workunits/ceph-helpers.sh:873: get_num_active_clean: wc -l ../qa/workunits/ceph-helpers.sh:993: wait_for_clean: cur_active_clean=0 ../qa/workunits/ceph-helpers.sh:994: wait_for_clean: test 0 '!=' 0 ../qa/workunits/ceph-helpers.sh:998: wait_for_clean: sleep 1 .../qa/workunits/ceph-helpers.sh:999: wait_for_clean: expr 43 + 1 ../qa/workunits/ceph-helpers.sh:999: wait_for_clean: timer=44 ../qa/workunits/ceph-helpers.sh:985: wait_for_clean: is_clean .../qa/workunits/ceph-helpers.sh:952: is_clean: get_num_pgs .../qa/workunits/ceph-helpers.sh:898: get_num_pgs: ceph --format xml status .../qa/workunits/ceph-helpers.sh:899: get_num_pgs: xmlstarlet sel -t -m //pgmap/num_pgs -v . ../qa/workunits/ceph-helpers.sh:952: is_clean: num_pgs=4 ../qa/workunits/ceph-helpers.sh:953: is_clean: test 4 '!=' 0 .../qa/workunits/ceph-helpers.sh:954: is_clean: get_num_active_clean .../qa/workunits/ceph-helpers.sh:863: get_num_active_clean: local 'expression=(' .../qa/workunits/ceph-helpers.sh:864: get_num_active_clean: expression+='contains(.,'\''active'\'') and ' .../qa/workunits/ceph-helpers.sh:865: get_num_active_clean: expression+='contains(.,'\''clean'\'') and ' .../qa/workunits/ceph-helpers.sh:866: get_num_active_clean: expression+='not(contains(.,'\''stale'\''))' .../qa/workunits/ceph-helpers.sh:867: get_num_active_clean: expression+=')' .../qa/workunits/ceph-helpers.sh:871: get_num_active_clean: ceph --format xml pg dump pgs .../qa/workunits/ceph-helpers.sh:872: get_num_active_clean: xmlstarlet sel -t -m '//pg_stat/state[(contains(.,'\''active'\'') and contains(.,'\''clean'\'') and not(contains(.,'\''stale'\'')))]' -v . -n .../qa/workunits/ceph-helpers.sh:873: get_num_active_clean: grep -v '^$' .../qa/workunits/ceph-helpers.sh:873: get_num_active_clean: wc -l ../qa/workunits/ceph-helpers.sh:954: is_clean: test 0 = 4 ../qa/workunits/ceph-helpers.sh:954: is_clean: return 1 ../qa/workunits/ceph-helpers.sh:986: wait_for_clean: get_is_making_recovery_progress .../qa/workunits/ceph-helpers.sh:839: get_is_making_recovery_progress: ceph --format xml status .../qa/workunits/ceph-helpers.sh:839: get_is_making_recovery_progress: xmlstarlet sel -t -m //pgmap/recovering_keys_per_sec -v . -o ' ' -t -m //pgmap/recovering_bytes_per_sec -v . -o ' ' -t -m //pgmap/recovering_objects_per_sec -v . ../qa/workunits/ceph-helpers.sh:839: get_is_making_recovery_progress: local progress= ../qa/workunits/ceph-helpers.sh:840: get_is_making_recovery_progress: test -n '' ../qa/workunits/ceph-helpers.sh:988: wait_for_clean: (( timer >= 120 )) .../qa/workunits/ceph-helpers.sh:993: wait_for_clean: get_num_active_clean .../qa/workunits/ceph-helpers.sh:863: get_num_active_clean: local 'expression=(' .../qa/workunits/ceph-helpers.sh:864: get_num_active_clean: expression+='contains(.,'\''active'\'') and ' .../qa/workunits/ceph-helpers.sh:865: get_num_active_clean: expression+='contains(.,'\''clean'\'') and ' .../qa/workunits/ceph-helpers.sh:866: get_num_active_clean: expression+='not(contains(.,'\''stale'\''))' .../qa/workunits/ceph-helpers.sh:867: get_num_active_clean: expression+=')' .../qa/workunits/ceph-helpers.sh:871: get_num_active_clean: ceph --format xml pg dump pgs .../qa/workunits/ceph-helpers.sh:872: get_num_active_clean: xmlstarlet sel -t -m '//pg_stat/state[(contains(.,'\''active'\'') and contains(.,'\''clean'\'') and not(contains(.,'\''stale'\'')))]' -v . -n .../qa/workunits/ceph-helpers.sh:873: get_num_active_clean: grep -v '^$' .../qa/workunits/ceph-helpers.sh:873: get_num_active_clean: wc -l ../qa/workunits/ceph-helpers.sh:993: wait_for_clean: cur_active_clean=0 ../qa/workunits/ceph-helpers.sh:994: wait_for_clean: test 0 '!=' 0 ../qa/workunits/ceph-helpers.sh:998: wait_for_clean: sleep 1 .../qa/workunits/ceph-helpers.sh:999: wait_for_clean: expr 44 + 1 ../qa/workunits/ceph-helpers.sh:999: wait_for_clean: timer=45 ../qa/workunits/ceph-helpers.sh:985: wait_for_clean: is_clean .../qa/workunits/ceph-helpers.sh:952: is_clean: get_num_pgs .../qa/workunits/ceph-helpers.sh:898: get_num_pgs: ceph --format xml status .../qa/workunits/ceph-helpers.sh:899: get_num_pgs: xmlstarlet sel -t -m //pgmap/num_pgs -v . ../qa/workunits/ceph-helpers.sh:952: is_clean: num_pgs=4 ../qa/workunits/ceph-helpers.sh:953: is_clean: test 4 '!=' 0 .../qa/workunits/ceph-helpers.sh:954: is_clean: get_num_active_clean .../qa/workunits/ceph-helpers.sh:863: get_num_active_clean: local 'expression=(' .../qa/workunits/ceph-helpers.sh:864: get_num_active_clean: expression+='contains(.,'\''active'\'') and ' .../qa/workunits/ceph-helpers.sh:865: get_num_active_clean: expression+='contains(.,'\''clean'\'') and ' .../qa/workunits/ceph-helpers.sh:866: get_num_active_clean: expression+='not(contains(.,'\''stale'\''))' .../qa/workunits/ceph-helpers.sh:867: get_num_active_clean: expression+=')' .../qa/workunits/ceph-helpers.sh:871: get_num_active_clean: ceph --format xml pg dump pgs .../qa/workunits/ceph-helpers.sh:872: get_num_active_clean: xmlstarlet sel -t -m '//pg_stat/state[(contains(.,'\''active'\'') and contains(.,'\''clean'\'') and not(contains(.,'\''stale'\'')))]' -v . -n .../qa/workunits/ceph-helpers.sh:873: get_num_active_clean: grep -v '^$' .../qa/workunits/ceph-helpers.sh:873: get_num_active_clean: wc -l ../qa/workunits/ceph-helpers.sh:954: is_clean: test 0 = 4 ../qa/workunits/ceph-helpers.sh:954: is_clean: return 1 ../qa/workunits/ceph-helpers.sh:986: wait_for_clean: get_is_making_recovery_progress .../qa/workunits/ceph-helpers.sh:839: get_is_making_recovery_progress: xmlstarlet sel -t -m //pgmap/recovering_keys_per_sec -v . -o ' ' -t -m //pgmap/recovering_bytes_per_sec -v . -o ' ' -t -m //pgmap/recovering_objects_per_sec -v . .../qa/workunits/ceph-helpers.sh:839: get_is_making_recovery_progress: ceph --format xml status ../qa/workunits/ceph-helpers.sh:839: get_is_making_recovery_progress: local progress= ../qa/workunits/ceph-helpers.sh:840: get_is_making_recovery_progress: test -n '' ../qa/workunits/ceph-helpers.sh:988: wait_for_clean: (( timer >= 120 )) .../qa/workunits/ceph-helpers.sh:993: wait_for_clean: get_num_active_clean .../qa/workunits/ceph-helpers.sh:863: get_num_active_clean: local 'expression=(' .../qa/workunits/ceph-helpers.sh:864: get_num_active_clean: expression+='contains(.,'\''active'\'') and ' .../qa/workunits/ceph-helpers.sh:865: get_num_active_clean: expression+='contains(.,'\''clean'\'') and ' .../qa/workunits/ceph-helpers.sh:866: get_num_active_clean: expression+='not(contains(.,'\''stale'\''))' .../qa/workunits/ceph-helpers.sh:867: get_num_active_clean: expression+=')' .../qa/workunits/ceph-helpers.sh:871: get_num_active_clean: ceph --format xml pg dump pgs .../qa/workunits/ceph-helpers.sh:872: get_num_active_clean: xmlstarlet sel -t -m '//pg_stat/state[(contains(.,'\''active'\'') and contains(.,'\''clean'\'') and not(contains(.,'\''stale'\'')))]' -v . -n .../qa/workunits/ceph-helpers.sh:873: get_num_active_clean: grep -v '^$' .../qa/workunits/ceph-helpers.sh:873: get_num_active_clean: wc -l ../qa/workunits/ceph-helpers.sh:993: wait_for_clean: cur_active_clean=0 ../qa/workunits/ceph-helpers.sh:994: wait_for_clean: test 0 '!=' 0 ../qa/workunits/ceph-helpers.sh:998: wait_for_clean: sleep 1 .../qa/workunits/ceph-helpers.sh:999: wait_for_clean: expr 45 + 1 ../qa/workunits/ceph-helpers.sh:999: wait_for_clean: timer=46 ../qa/workunits/ceph-helpers.sh:985: wait_for_clean: is_clean .../qa/workunits/ceph-helpers.sh:952: is_clean: get_num_pgs .../qa/workunits/ceph-helpers.sh:898: get_num_pgs: ceph --format xml status .../qa/workunits/ceph-helpers.sh:899: get_num_pgs: xmlstarlet sel -t -m //pgmap/num_pgs -v . ../qa/workunits/ceph-helpers.sh:952: is_clean: num_pgs=4 ../qa/workunits/ceph-helpers.sh:953: is_clean: test 4 '!=' 0 .../qa/workunits/ceph-helpers.sh:954: is_clean: get_num_active_clean .../qa/workunits/ceph-helpers.sh:863: get_num_active_clean: local 'expression=(' .../qa/workunits/ceph-helpers.sh:864: get_num_active_clean: expression+='contains(.,'\''active'\'') and ' .../qa/workunits/ceph-helpers.sh:865: get_num_active_clean: expression+='contains(.,'\''clean'\'') and ' .../qa/workunits/ceph-helpers.sh:866: get_num_active_clean: expression+='not(contains(.,'\''stale'\''))' .../qa/workunits/ceph-helpers.sh:867: get_num_active_clean: expression+=')' .../qa/workunits/ceph-helpers.sh:871: get_num_active_clean: ceph --format xml pg dump pgs .../qa/workunits/ceph-helpers.sh:872: get_num_active_clean: xmlstarlet sel -t -m '//pg_stat/state[(contains(.,'\''active'\'') and contains(.,'\''clean'\'') and not(contains(.,'\''stale'\'')))]' -v . -n .../qa/workunits/ceph-helpers.sh:873: get_num_active_clean: grep -v '^$' .../qa/workunits/ceph-helpers.sh:873: get_num_active_clean: wc -l ../qa/workunits/ceph-helpers.sh:954: is_clean: test 0 = 4 ../qa/workunits/ceph-helpers.sh:954: is_clean: return 1 ../qa/workunits/ceph-helpers.sh:986: wait_for_clean: get_is_making_recovery_progress .../qa/workunits/ceph-helpers.sh:839: get_is_making_recovery_progress: ceph --format xml status .../qa/workunits/ceph-helpers.sh:839: get_is_making_recovery_progress: xmlstarlet sel -t -m //pgmap/recovering_keys_per_sec -v . -o ' ' -t -m //pgmap/recovering_bytes_per_sec -v . -o ' ' -t -m //pgmap/recovering_objects_per_sec -v . ../qa/workunits/ceph-helpers.sh:839: get_is_making_recovery_progress: local progress= ../qa/workunits/ceph-helpers.sh:840: get_is_making_recovery_progress: test -n '' ../qa/workunits/ceph-helpers.sh:988: wait_for_clean: (( timer >= 120 )) .../qa/workunits/ceph-helpers.sh:993: wait_for_clean: get_num_active_clean .../qa/workunits/ceph-helpers.sh:863: get_num_active_clean: local 'expression=(' .../qa/workunits/ceph-helpers.sh:864: get_num_active_clean: expression+='contains(.,'\''active'\'') and ' .../qa/workunits/ceph-helpers.sh:865: get_num_active_clean: expression+='contains(.,'\''clean'\'') and ' .../qa/workunits/ceph-helpers.sh:866: get_num_active_clean: expression+='not(contains(.,'\''stale'\''))' .../qa/workunits/ceph-helpers.sh:867: get_num_active_clean: expression+=')' .../qa/workunits/ceph-helpers.sh:871: get_num_active_clean: ceph --format xml pg dump pgs .../qa/workunits/ceph-helpers.sh:872: get_num_active_clean: xmlstarlet sel -t -m '//pg_stat/state[(contains(.,'\''active'\'') and contains(.,'\''clean'\'') and not(contains(.,'\''stale'\'')))]' -v . -n .../qa/workunits/ceph-helpers.sh:873: get_num_active_clean: grep -v '^$' .../qa/workunits/ceph-helpers.sh:873: get_num_active_clean: wc -l ../qa/workunits/ceph-helpers.sh:993: wait_for_clean: cur_active_clean=0 ../qa/workunits/ceph-helpers.sh:994: wait_for_clean: test 0 '!=' 0 ../qa/workunits/ceph-helpers.sh:998: wait_for_clean: sleep 1 .../qa/workunits/ceph-helpers.sh:999: wait_for_clean: expr 46 + 1 ../qa/workunits/ceph-helpers.sh:999: wait_for_clean: timer=47 ../qa/workunits/ceph-helpers.sh:985: wait_for_clean: is_clean .../qa/workunits/ceph-helpers.sh:952: is_clean: get_num_pgs .../qa/workunits/ceph-helpers.sh:898: get_num_pgs: ceph --format xml status .../qa/workunits/ceph-helpers.sh:899: get_num_pgs: xmlstarlet sel -t -m //pgmap/num_pgs -v . ../qa/workunits/ceph-helpers.sh:952: is_clean: num_pgs=4 ../qa/workunits/ceph-helpers.sh:953: is_clean: test 4 '!=' 0 .../qa/workunits/ceph-helpers.sh:954: is_clean: get_num_active_clean .../qa/workunits/ceph-helpers.sh:863: get_num_active_clean: local 'expression=(' .../qa/workunits/ceph-helpers.sh:864: get_num_active_clean: expression+='contains(.,'\''active'\'') and ' .../qa/workunits/ceph-helpers.sh:865: get_num_active_clean: expression+='contains(.,'\''clean'\'') and ' .../qa/workunits/ceph-helpers.sh:866: get_num_active_clean: expression+='not(contains(.,'\''stale'\''))' .../qa/workunits/ceph-helpers.sh:867: get_num_active_clean: expression+=')' .../qa/workunits/ceph-helpers.sh:871: get_num_active_clean: ceph --format xml pg dump pgs .../qa/workunits/ceph-helpers.sh:872: get_num_active_clean: xmlstarlet sel -t -m '//pg_stat/state[(contains(.,'\''active'\'') and contains(.,'\''clean'\'') and not(contains(.,'\''stale'\'')))]' -v . -n .../qa/workunits/ceph-helpers.sh:873: get_num_active_clean: grep -v '^$' .../qa/workunits/ceph-helpers.sh:873: get_num_active_clean: wc -l ../qa/workunits/ceph-helpers.sh:954: is_clean: test 0 = 4 ../qa/workunits/ceph-helpers.sh:954: is_clean: return 1 ../qa/workunits/ceph-helpers.sh:986: wait_for_clean: get_is_making_recovery_progress .../qa/workunits/ceph-helpers.sh:839: get_is_making_recovery_progress: ceph --format xml status .../qa/workunits/ceph-helpers.sh:839: get_is_making_recovery_progress: xmlstarlet sel -t -m //pgmap/recovering_keys_per_sec -v . -o ' ' -t -m //pgmap/recovering_bytes_per_sec -v . -o ' ' -t -m //pgmap/recovering_objects_per_sec -v . ../qa/workunits/ceph-helpers.sh:839: get_is_making_recovery_progress: local progress= ../qa/workunits/ceph-helpers.sh:840: get_is_making_recovery_progress: test -n '' ../qa/workunits/ceph-helpers.sh:988: wait_for_clean: (( timer >= 120 )) .../qa/workunits/ceph-helpers.sh:993: wait_for_clean: get_num_active_clean .../qa/workunits/ceph-helpers.sh:863: get_num_active_clean: local 'expression=(' .../qa/workunits/ceph-helpers.sh:864: get_num_active_clean: expression+='contains(.,'\''active'\'') and ' .../qa/workunits/ceph-helpers.sh:865: get_num_active_clean: expression+='contains(.,'\''clean'\'') and ' .../qa/workunits/ceph-helpers.sh:866: get_num_active_clean: expression+='not(contains(.,'\''stale'\''))' .../qa/workunits/ceph-helpers.sh:867: get_num_active_clean: expression+=')' .../qa/workunits/ceph-helpers.sh:871: get_num_active_clean: ceph --format xml pg dump pgs .../qa/workunits/ceph-helpers.sh:872: get_num_active_clean: xmlstarlet sel -t -m '//pg_stat/state[(contains(.,'\''active'\'') and contains(.,'\''clean'\'') and not(contains(.,'\''stale'\'')))]' -v . -n .../qa/workunits/ceph-helpers.sh:873: get_num_active_clean: grep -v '^$' .../qa/workunits/ceph-helpers.sh:873: get_num_active_clean: wc -l ../qa/workunits/ceph-helpers.sh:993: wait_for_clean: cur_active_clean=0 ../qa/workunits/ceph-helpers.sh:994: wait_for_clean: test 0 '!=' 0 ../qa/workunits/ceph-helpers.sh:998: wait_for_clean: sleep 1 .../qa/workunits/ceph-helpers.sh:999: wait_for_clean: expr 47 + 1 ../qa/workunits/ceph-helpers.sh:999: wait_for_clean: timer=48 ../qa/workunits/ceph-helpers.sh:985: wait_for_clean: is_clean .../qa/workunits/ceph-helpers.sh:952: is_clean: get_num_pgs .../qa/workunits/ceph-helpers.sh:898: get_num_pgs: ceph --format xml status .../qa/workunits/ceph-helpers.sh:899: get_num_pgs: xmlstarlet sel -t -m //pgmap/num_pgs -v . ../qa/workunits/ceph-helpers.sh:952: is_clean: num_pgs=4 ../qa/workunits/ceph-helpers.sh:953: is_clean: test 4 '!=' 0 .../qa/workunits/ceph-helpers.sh:954: is_clean: get_num_active_clean .../qa/workunits/ceph-helpers.sh:863: get_num_active_clean: local 'expression=(' .../qa/workunits/ceph-helpers.sh:864: get_num_active_clean: expression+='contains(.,'\''active'\'') and ' .../qa/workunits/ceph-helpers.sh:865: get_num_active_clean: expression+='contains(.,'\''clean'\'') and ' .../qa/workunits/ceph-helpers.sh:866: get_num_active_clean: expression+='not(contains(.,'\''stale'\''))' .../qa/workunits/ceph-helpers.sh:867: get_num_active_clean: expression+=')' .../qa/workunits/ceph-helpers.sh:871: get_num_active_clean: ceph --format xml pg dump pgs .../qa/workunits/ceph-helpers.sh:872: get_num_active_clean: xmlstarlet sel -t -m '//pg_stat/state[(contains(.,'\''active'\'') and contains(.,'\''clean'\'') and not(contains(.,'\''stale'\'')))]' -v . -n .../qa/workunits/ceph-helpers.sh:873: get_num_active_clean: grep -v '^$' .../qa/workunits/ceph-helpers.sh:873: get_num_active_clean: wc -l ../qa/workunits/ceph-helpers.sh:954: is_clean: test 0 = 4 ../qa/workunits/ceph-helpers.sh:954: is_clean: return 1 ../qa/workunits/ceph-helpers.sh:986: wait_for_clean: get_is_making_recovery_progress .../qa/workunits/ceph-helpers.sh:839: get_is_making_recovery_progress: ceph --format xml status .../qa/workunits/ceph-helpers.sh:839: get_is_making_recovery_progress: xmlstarlet sel -t -m //pgmap/recovering_keys_per_sec -v . -o ' ' -t -m //pgmap/recovering_bytes_per_sec -v . -o ' ' -t -m //pgmap/recovering_objects_per_sec -v . ../qa/workunits/ceph-helpers.sh:839: get_is_making_recovery_progress: local progress= ../qa/workunits/ceph-helpers.sh:840: get_is_making_recovery_progress: test -n '' ../qa/workunits/ceph-helpers.sh:988: wait_for_clean: (( timer >= 120 )) .../qa/workunits/ceph-helpers.sh:993: wait_for_clean: get_num_active_clean .../qa/workunits/ceph-helpers.sh:863: get_num_active_clean: local 'expression=(' .../qa/workunits/ceph-helpers.sh:864: get_num_active_clean: expression+='contains(.,'\''active'\'') and ' .../qa/workunits/ceph-helpers.sh:865: get_num_active_clean: expression+='contains(.,'\''clean'\'') and ' .../qa/workunits/ceph-helpers.sh:866: get_num_active_clean: expression+='not(contains(.,'\''stale'\''))' .../qa/workunits/ceph-helpers.sh:867: get_num_active_clean: expression+=')' .../qa/workunits/ceph-helpers.sh:871: get_num_active_clean: ceph --format xml pg dump pgs .../qa/workunits/ceph-helpers.sh:872: get_num_active_clean: xmlstarlet sel -t -m '//pg_stat/state[(contains(.,'\''active'\'') and contains(.,'\''clean'\'') and not(contains(.,'\''stale'\'')))]' -v . -n .../qa/workunits/ceph-helpers.sh:873: get_num_active_clean: grep -v '^$' .../qa/workunits/ceph-helpers.sh:873: get_num_active_clean: wc -l ../qa/workunits/ceph-helpers.sh:993: wait_for_clean: cur_active_clean=0 ../qa/workunits/ceph-helpers.sh:994: wait_for_clean: test 0 '!=' 0 ../qa/workunits/ceph-helpers.sh:998: wait_for_clean: sleep 1 .../qa/workunits/ceph-helpers.sh:999: wait_for_clean: expr 48 + 1 ../qa/workunits/ceph-helpers.sh:999: wait_for_clean: timer=49 ../qa/workunits/ceph-helpers.sh:985: wait_for_clean: is_clean .../qa/workunits/ceph-helpers.sh:952: is_clean: get_num_pgs .../qa/workunits/ceph-helpers.sh:898: get_num_pgs: ceph --format xml status .../qa/workunits/ceph-helpers.sh:899: get_num_pgs: xmlstarlet sel -t -m //pgmap/num_pgs -v . ../qa/workunits/ceph-helpers.sh:952: is_clean: num_pgs=4 ../qa/workunits/ceph-helpers.sh:953: is_clean: test 4 '!=' 0 .../qa/workunits/ceph-helpers.sh:954: is_clean: get_num_active_clean .../qa/workunits/ceph-helpers.sh:863: get_num_active_clean: local 'expression=(' .../qa/workunits/ceph-helpers.sh:864: get_num_active_clean: expression+='contains(.,'\''active'\'') and ' .../qa/workunits/ceph-helpers.sh:865: get_num_active_clean: expression+='contains(.,'\''clean'\'') and ' .../qa/workunits/ceph-helpers.sh:866: get_num_active_clean: expression+='not(contains(.,'\''stale'\''))' .../qa/workunits/ceph-helpers.sh:867: get_num_active_clean: expression+=')' .../qa/workunits/ceph-helpers.sh:871: get_num_active_clean: ceph --format xml pg dump pgs .../qa/workunits/ceph-helpers.sh:872: get_num_active_clean: xmlstarlet sel -t -m '//pg_stat/state[(contains(.,'\''active'\'') and contains(.,'\''clean'\'') and not(contains(.,'\''stale'\'')))]' -v . -n .../qa/workunits/ceph-helpers.sh:873: get_num_active_clean: wc -l .../qa/workunits/ceph-helpers.sh:873: get_num_active_clean: grep -v '^$' ../qa/workunits/ceph-helpers.sh:954: is_clean: test 0 = 4 ../qa/workunits/ceph-helpers.sh:954: is_clean: return 1 ../qa/workunits/ceph-helpers.sh:986: wait_for_clean: get_is_making_recovery_progress .../qa/workunits/ceph-helpers.sh:839: get_is_making_recovery_progress: ceph --format xml status .../qa/workunits/ceph-helpers.sh:839: get_is_making_recovery_progress: xmlstarlet sel -t -m //pgmap/recovering_keys_per_sec -v . -o ' ' -t -m //pgmap/recovering_bytes_per_sec -v . -o ' ' -t -m //pgmap/recovering_objects_per_sec -v . ../qa/workunits/ceph-helpers.sh:839: get_is_making_recovery_progress: local progress= ../qa/workunits/ceph-helpers.sh:840: get_is_making_recovery_progress: test -n '' ../qa/workunits/ceph-helpers.sh:988: wait_for_clean: (( timer >= 120 )) .../qa/workunits/ceph-helpers.sh:993: wait_for_clean: get_num_active_clean .../qa/workunits/ceph-helpers.sh:863: get_num_active_clean: local 'expression=(' .../qa/workunits/ceph-helpers.sh:864: get_num_active_clean: expression+='contains(.,'\''active'\'') and ' .../qa/workunits/ceph-helpers.sh:865: get_num_active_clean: expression+='contains(.,'\''clean'\'') and ' .../qa/workunits/ceph-helpers.sh:866: get_num_active_clean: expression+='not(contains(.,'\''stale'\''))' .../qa/workunits/ceph-helpers.sh:867: get_num_active_clean: expression+=')' .../qa/workunits/ceph-helpers.sh:871: get_num_active_clean: ceph --format xml pg dump pgs .../qa/workunits/ceph-helpers.sh:872: get_num_active_clean: xmlstarlet sel -t -m '//pg_stat/state[(contains(.,'\''active'\'') and contains(.,'\''clean'\'') and not(contains(.,'\''stale'\'')))]' -v . -n .../qa/workunits/ceph-helpers.sh:873: get_num_active_clean: grep -v '^$' .../qa/workunits/ceph-helpers.sh:873: get_num_active_clean: wc -l ../qa/workunits/ceph-helpers.sh:993: wait_for_clean: cur_active_clean=0 ../qa/workunits/ceph-helpers.sh:994: wait_for_clean: test 0 '!=' 0 ../qa/workunits/ceph-helpers.sh:998: wait_for_clean: sleep 1 .../qa/workunits/ceph-helpers.sh:999: wait_for_clean: expr 49 + 1 ../qa/workunits/ceph-helpers.sh:999: wait_for_clean: timer=50 ../qa/workunits/ceph-helpers.sh:985: wait_for_clean: is_clean .../qa/workunits/ceph-helpers.sh:952: is_clean: get_num_pgs .../qa/workunits/ceph-helpers.sh:898: get_num_pgs: ceph --format xml status .../qa/workunits/ceph-helpers.sh:899: get_num_pgs: xmlstarlet sel -t -m //pgmap/num_pgs -v . ../qa/workunits/ceph-helpers.sh:952: is_clean: num_pgs=4 ../qa/workunits/ceph-helpers.sh:953: is_clean: test 4 '!=' 0 .../qa/workunits/ceph-helpers.sh:954: is_clean: get_num_active_clean .../qa/workunits/ceph-helpers.sh:863: get_num_active_clean: local 'expression=(' .../qa/workunits/ceph-helpers.sh:864: get_num_active_clean: expression+='contains(.,'\''active'\'') and ' .../qa/workunits/ceph-helpers.sh:865: get_num_active_clean: expression+='contains(.,'\''clean'\'') and ' .../qa/workunits/ceph-helpers.sh:866: get_num_active_clean: expression+='not(contains(.,'\''stale'\''))' .../qa/workunits/ceph-helpers.sh:867: get_num_active_clean: expression+=')' .../qa/workunits/ceph-helpers.sh:871: get_num_active_clean: ceph --format xml pg dump pgs .../qa/workunits/ceph-helpers.sh:872: get_num_active_clean: xmlstarlet sel -t -m '//pg_stat/state[(contains(.,'\''active'\'') and contains(.,'\''clean'\'') and not(contains(.,'\''stale'\'')))]' -v . -n .../qa/workunits/ceph-helpers.sh:873: get_num_active_clean: grep -v '^$' .../qa/workunits/ceph-helpers.sh:873: get_num_active_clean: wc -l ../qa/workunits/ceph-helpers.sh:954: is_clean: test 0 = 4 ../qa/workunits/ceph-helpers.sh:954: is_clean: return 1 ../qa/workunits/ceph-helpers.sh:986: wait_for_clean: get_is_making_recovery_progress .../qa/workunits/ceph-helpers.sh:839: get_is_making_recovery_progress: ceph --format xml status .../qa/workunits/ceph-helpers.sh:839: get_is_making_recovery_progress: xmlstarlet sel -t -m //pgmap/recovering_keys_per_sec -v . -o ' ' -t -m //pgmap/recovering_bytes_per_sec -v . -o ' ' -t -m //pgmap/recovering_objects_per_sec -v . ../qa/workunits/ceph-helpers.sh:839: get_is_making_recovery_progress: local progress= ../qa/workunits/ceph-helpers.sh:840: get_is_making_recovery_progress: test -n '' ../qa/workunits/ceph-helpers.sh:988: wait_for_clean: (( timer >= 120 )) .../qa/workunits/ceph-helpers.sh:993: wait_for_clean: get_num_active_clean .../qa/workunits/ceph-helpers.sh:863: get_num_active_clean: local 'expression=(' .../qa/workunits/ceph-helpers.sh:864: get_num_active_clean: expression+='contains(.,'\''active'\'') and ' .../qa/workunits/ceph-helpers.sh:865: get_num_active_clean: expression+='contains(.,'\''clean'\'') and ' .../qa/workunits/ceph-helpers.sh:866: get_num_active_clean: expression+='not(contains(.,'\''stale'\''))' .../qa/workunits/ceph-helpers.sh:867: get_num_active_clean: expression+=')' .../qa/workunits/ceph-helpers.sh:871: get_num_active_clean: ceph --format xml pg dump pgs .../qa/workunits/ceph-helpers.sh:872: get_num_active_clean: xmlstarlet sel -t -m '//pg_stat/state[(contains(.,'\''active'\'') and contains(.,'\''clean'\'') and not(contains(.,'\''stale'\'')))]' -v . -n .../qa/workunits/ceph-helpers.sh:873: get_num_active_clean: grep -v '^$' .../qa/workunits/ceph-helpers.sh:873: get_num_active_clean: wc -l ../qa/workunits/ceph-helpers.sh:993: wait_for_clean: cur_active_clean=0 ../qa/workunits/ceph-helpers.sh:994: wait_for_clean: test 0 '!=' 0 ../qa/workunits/ceph-helpers.sh:998: wait_for_clean: sleep 1 .../qa/workunits/ceph-helpers.sh:999: wait_for_clean: expr 50 + 1 ../qa/workunits/ceph-helpers.sh:999: wait_for_clean: timer=51 ../qa/workunits/ceph-helpers.sh:985: wait_for_clean: is_clean .../qa/workunits/ceph-helpers.sh:952: is_clean: get_num_pgs .../qa/workunits/ceph-helpers.sh:898: get_num_pgs: ceph --format xml status .../qa/workunits/ceph-helpers.sh:899: get_num_pgs: xmlstarlet sel -t -m //pgmap/num_pgs -v . ../qa/workunits/ceph-helpers.sh:952: is_clean: num_pgs=4 ../qa/workunits/ceph-helpers.sh:953: is_clean: test 4 '!=' 0 .../qa/workunits/ceph-helpers.sh:954: is_clean: get_num_active_clean .../qa/workunits/ceph-helpers.sh:863: get_num_active_clean: local 'expression=(' .../qa/workunits/ceph-helpers.sh:864: get_num_active_clean: expression+='contains(.,'\''active'\'') and ' .../qa/workunits/ceph-helpers.sh:865: get_num_active_clean: expression+='contains(.,'\''clean'\'') and ' .../qa/workunits/ceph-helpers.sh:866: get_num_active_clean: expression+='not(contains(.,'\''stale'\''))' .../qa/workunits/ceph-helpers.sh:867: get_num_active_clean: expression+=')' .../qa/workunits/ceph-helpers.sh:871: get_num_active_clean: ceph --format xml pg dump pgs .../qa/workunits/ceph-helpers.sh:872: get_num_active_clean: xmlstarlet sel -t -m '//pg_stat/state[(contains(.,'\''active'\'') and contains(.,'\''clean'\'') and not(contains(.,'\''stale'\'')))]' -v . -n .../qa/workunits/ceph-helpers.sh:873: get_num_active_clean: grep -v '^$' .../qa/workunits/ceph-helpers.sh:873: get_num_active_clean: wc -l ../qa/workunits/ceph-helpers.sh:954: is_clean: test 0 = 4 ../qa/workunits/ceph-helpers.sh:954: is_clean: return 1 ../qa/workunits/ceph-helpers.sh:986: wait_for_clean: get_is_making_recovery_progress .../qa/workunits/ceph-helpers.sh:839: get_is_making_recovery_progress: ceph --format xml status .../qa/workunits/ceph-helpers.sh:839: get_is_making_recovery_progress: xmlstarlet sel -t -m //pgmap/recovering_keys_per_sec -v . -o ' ' -t -m //pgmap/recovering_bytes_per_sec -v . -o ' ' -t -m //pgmap/recovering_objects_per_sec -v . ../qa/workunits/ceph-helpers.sh:839: get_is_making_recovery_progress: local progress= ../qa/workunits/ceph-helpers.sh:840: get_is_making_recovery_progress: test -n '' ../qa/workunits/ceph-helpers.sh:988: wait_for_clean: (( timer >= 120 )) .../qa/workunits/ceph-helpers.sh:993: wait_for_clean: get_num_active_clean .../qa/workunits/ceph-helpers.sh:863: get_num_active_clean: local 'expression=(' .../qa/workunits/ceph-helpers.sh:864: get_num_active_clean: expression+='contains(.,'\''active'\'') and ' .../qa/workunits/ceph-helpers.sh:865: get_num_active_clean: expression+='contains(.,'\''clean'\'') and ' .../qa/workunits/ceph-helpers.sh:866: get_num_active_clean: expression+='not(contains(.,'\''stale'\''))' .../qa/workunits/ceph-helpers.sh:867: get_num_active_clean: expression+=')' .../qa/workunits/ceph-helpers.sh:871: get_num_active_clean: ceph --format xml pg dump pgs .../qa/workunits/ceph-helpers.sh:872: get_num_active_clean: xmlstarlet sel -t -m '//pg_stat/state[(contains(.,'\''active'\'') and contains(.,'\''clean'\'') and not(contains(.,'\''stale'\'')))]' -v . -n .../qa/workunits/ceph-helpers.sh:873: get_num_active_clean: grep -v '^$' .../qa/workunits/ceph-helpers.sh:873: get_num_active_clean: wc -l ../qa/workunits/ceph-helpers.sh:993: wait_for_clean: cur_active_clean=0 ../qa/workunits/ceph-helpers.sh:994: wait_for_clean: test 0 '!=' 0 ../qa/workunits/ceph-helpers.sh:998: wait_for_clean: sleep 1 .../qa/workunits/ceph-helpers.sh:999: wait_for_clean: expr 51 + 1 ../qa/workunits/ceph-helpers.sh:999: wait_for_clean: timer=52 ../qa/workunits/ceph-helpers.sh:985: wait_for_clean: is_clean .../qa/workunits/ceph-helpers.sh:952: is_clean: get_num_pgs .../qa/workunits/ceph-helpers.sh:898: get_num_pgs: ceph --format xml status .../qa/workunits/ceph-helpers.sh:899: get_num_pgs: xmlstarlet sel -t -m //pgmap/num_pgs -v . ../qa/workunits/ceph-helpers.sh:952: is_clean: num_pgs=4 ../qa/workunits/ceph-helpers.sh:953: is_clean: test 4 '!=' 0 .../qa/workunits/ceph-helpers.sh:954: is_clean: get_num_active_clean .../qa/workunits/ceph-helpers.sh:863: get_num_active_clean: local 'expression=(' .../qa/workunits/ceph-helpers.sh:864: get_num_active_clean: expression+='contains(.,'\''active'\'') and ' .../qa/workunits/ceph-helpers.sh:865: get_num_active_clean: expression+='contains(.,'\''clean'\'') and ' .../qa/workunits/ceph-helpers.sh:866: get_num_active_clean: expression+='not(contains(.,'\''stale'\''))' .../qa/workunits/ceph-helpers.sh:867: get_num_active_clean: expression+=')' .../qa/workunits/ceph-helpers.sh:871: get_num_active_clean: ceph --format xml pg dump pgs .../qa/workunits/ceph-helpers.sh:872: get_num_active_clean: xmlstarlet sel -t -m '//pg_stat/state[(contains(.,'\''active'\'') and contains(.,'\''clean'\'') and not(contains(.,'\''stale'\'')))]' -v . -n .../qa/workunits/ceph-helpers.sh:873: get_num_active_clean: wc -l .../qa/workunits/ceph-helpers.sh:873: get_num_active_clean: grep -v '^$' ../qa/workunits/ceph-helpers.sh:954: is_clean: test 0 = 4 ../qa/workunits/ceph-helpers.sh:954: is_clean: return 1 ../qa/workunits/ceph-helpers.sh:986: wait_for_clean: get_is_making_recovery_progress .../qa/workunits/ceph-helpers.sh:839: get_is_making_recovery_progress: ceph --format xml status .../qa/workunits/ceph-helpers.sh:839: get_is_making_recovery_progress: xmlstarlet sel -t -m //pgmap/recovering_keys_per_sec -v . -o ' ' -t -m //pgmap/recovering_bytes_per_sec -v . -o ' ' -t -m //pgmap/recovering_objects_per_sec -v . ../qa/workunits/ceph-helpers.sh:839: get_is_making_recovery_progress: local progress= ../qa/workunits/ceph-helpers.sh:840: get_is_making_recovery_progress: test -n '' ../qa/workunits/ceph-helpers.sh:988: wait_for_clean: (( timer >= 120 )) .../qa/workunits/ceph-helpers.sh:993: wait_for_clean: get_num_active_clean .../qa/workunits/ceph-helpers.sh:863: get_num_active_clean: local 'expression=(' .../qa/workunits/ceph-helpers.sh:864: get_num_active_clean: expression+='contains(.,'\''active'\'') and ' .../qa/workunits/ceph-helpers.sh:865: get_num_active_clean: expression+='contains(.,'\''clean'\'') and ' .../qa/workunits/ceph-helpers.sh:866: get_num_active_clean: expression+='not(contains(.,'\''stale'\''))' .../qa/workunits/ceph-helpers.sh:867: get_num_active_clean: expression+=')' .../qa/workunits/ceph-helpers.sh:871: get_num_active_clean: ceph --format xml pg dump pgs .../qa/workunits/ceph-helpers.sh:872: get_num_active_clean: xmlstarlet sel -t -m '//pg_stat/state[(contains(.,'\''active'\'') and contains(.,'\''clean'\'') and not(contains(.,'\''stale'\'')))]' -v . -n .../qa/workunits/ceph-helpers.sh:873: get_num_active_clean: grep -v '^$' .../qa/workunits/ceph-helpers.sh:873: get_num_active_clean: wc -l ../qa/workunits/ceph-helpers.sh:993: wait_for_clean: cur_active_clean=0 ../qa/workunits/ceph-helpers.sh:994: wait_for_clean: test 0 '!=' 0 ../qa/workunits/ceph-helpers.sh:998: wait_for_clean: sleep 1 .../qa/workunits/ceph-helpers.sh:999: wait_for_clean: expr 52 + 1 ../qa/workunits/ceph-helpers.sh:999: wait_for_clean: timer=53 ../qa/workunits/ceph-helpers.sh:985: wait_for_clean: is_clean .../qa/workunits/ceph-helpers.sh:952: is_clean: get_num_pgs .../qa/workunits/ceph-helpers.sh:898: get_num_pgs: ceph --format xml status .../qa/workunits/ceph-helpers.sh:899: get_num_pgs: xmlstarlet sel -t -m //pgmap/num_pgs -v . ../qa/workunits/ceph-helpers.sh:952: is_clean: num_pgs=4 ../qa/workunits/ceph-helpers.sh:953: is_clean: test 4 '!=' 0 .../qa/workunits/ceph-helpers.sh:954: is_clean: get_num_active_clean .../qa/workunits/ceph-helpers.sh:863: get_num_active_clean: local 'expression=(' .../qa/workunits/ceph-helpers.sh:864: get_num_active_clean: expression+='contains(.,'\''active'\'') and ' .../qa/workunits/ceph-helpers.sh:865: get_num_active_clean: expression+='contains(.,'\''clean'\'') and ' .../qa/workunits/ceph-helpers.sh:866: get_num_active_clean: expression+='not(contains(.,'\''stale'\''))' .../qa/workunits/ceph-helpers.sh:867: get_num_active_clean: expression+=')' .../qa/workunits/ceph-helpers.sh:871: get_num_active_clean: ceph --format xml pg dump pgs .../qa/workunits/ceph-helpers.sh:872: get_num_active_clean: xmlstarlet sel -t -m '//pg_stat/state[(contains(.,'\''active'\'') and contains(.,'\''clean'\'') and not(contains(.,'\''stale'\'')))]' -v . -n .../qa/workunits/ceph-helpers.sh:873: get_num_active_clean: wc -l .../qa/workunits/ceph-helpers.sh:873: get_num_active_clean: grep -v '^$' ../qa/workunits/ceph-helpers.sh:954: is_clean: test 0 = 4 ../qa/workunits/ceph-helpers.sh:954: is_clean: return 1 ../qa/workunits/ceph-helpers.sh:986: wait_for_clean: get_is_making_recovery_progress .../qa/workunits/ceph-helpers.sh:839: get_is_making_recovery_progress: ceph --format xml status .../qa/workunits/ceph-helpers.sh:839: get_is_making_recovery_progress: xmlstarlet sel -t -m //pgmap/recovering_keys_per_sec -v . -o ' ' -t -m //pgmap/recovering_bytes_per_sec -v . -o ' ' -t -m //pgmap/recovering_objects_per_sec -v . ../qa/workunits/ceph-helpers.sh:839: get_is_making_recovery_progress: local progress= ../qa/workunits/ceph-helpers.sh:840: get_is_making_recovery_progress: test -n '' ../qa/workunits/ceph-helpers.sh:988: wait_for_clean: (( timer >= 120 )) .../qa/workunits/ceph-helpers.sh:993: wait_for_clean: get_num_active_clean .../qa/workunits/ceph-helpers.sh:863: get_num_active_clean: local 'expression=(' .../qa/workunits/ceph-helpers.sh:864: get_num_active_clean: expression+='contains(.,'\''active'\'') and ' .../qa/workunits/ceph-helpers.sh:865: get_num_active_clean: expression+='contains(.,'\''clean'\'') and ' .../qa/workunits/ceph-helpers.sh:866: get_num_active_clean: expression+='not(contains(.,'\''stale'\''))' .../qa/workunits/ceph-helpers.sh:867: get_num_active_clean: expression+=')' .../qa/workunits/ceph-helpers.sh:871: get_num_active_clean: ceph --format xml pg dump pgs .../qa/workunits/ceph-helpers.sh:872: get_num_active_clean: xmlstarlet sel -t -m '//pg_stat/state[(contains(.,'\''active'\'') and contains(.,'\''clean'\'') and not(contains(.,'\''stale'\'')))]' -v . -n .../qa/workunits/ceph-helpers.sh:873: get_num_active_clean: wc -l .../qa/workunits/ceph-helpers.sh:873: get_num_active_clean: grep -v '^$' ../qa/workunits/ceph-helpers.sh:993: wait_for_clean: cur_active_clean=0 ../qa/workunits/ceph-helpers.sh:994: wait_for_clean: test 0 '!=' 0 ../qa/workunits/ceph-helpers.sh:998: wait_for_clean: sleep 1 .../qa/workunits/ceph-helpers.sh:999: wait_for_clean: expr 53 + 1 ../qa/workunits/ceph-helpers.sh:999: wait_for_clean: timer=54 ../qa/workunits/ceph-helpers.sh:985: wait_for_clean: is_clean .../qa/workunits/ceph-helpers.sh:952: is_clean: get_num_pgs .../qa/workunits/ceph-helpers.sh:898: get_num_pgs: ceph --format xml status .../qa/workunits/ceph-helpers.sh:899: get_num_pgs: xmlstarlet sel -t -m //pgmap/num_pgs -v . ../qa/workunits/ceph-helpers.sh:952: is_clean: num_pgs=4 ../qa/workunits/ceph-helpers.sh:953: is_clean: test 4 '!=' 0 .../qa/workunits/ceph-helpers.sh:954: is_clean: get_num_active_clean .../qa/workunits/ceph-helpers.sh:863: get_num_active_clean: local 'expression=(' .../qa/workunits/ceph-helpers.sh:864: get_num_active_clean: expression+='contains(.,'\''active'\'') and ' .../qa/workunits/ceph-helpers.sh:865: get_num_active_clean: expression+='contains(.,'\''clean'\'') and ' .../qa/workunits/ceph-helpers.sh:866: get_num_active_clean: expression+='not(contains(.,'\''stale'\''))' .../qa/workunits/ceph-helpers.sh:867: get_num_active_clean: expression+=')' .../qa/workunits/ceph-helpers.sh:871: get_num_active_clean: ceph --format xml pg dump pgs .../qa/workunits/ceph-helpers.sh:872: get_num_active_clean: xmlstarlet sel -t -m '//pg_stat/state[(contains(.,'\''active'\'') and contains(.,'\''clean'\'') and not(contains(.,'\''stale'\'')))]' -v . -n .../qa/workunits/ceph-helpers.sh:873: get_num_active_clean: grep -v '^$' .../qa/workunits/ceph-helpers.sh:873: get_num_active_clean: wc -l ../qa/workunits/ceph-helpers.sh:954: is_clean: test 0 = 4 ../qa/workunits/ceph-helpers.sh:954: is_clean: return 1 ../qa/workunits/ceph-helpers.sh:986: wait_for_clean: get_is_making_recovery_progress .../qa/workunits/ceph-helpers.sh:839: get_is_making_recovery_progress: ceph --format xml status .../qa/workunits/ceph-helpers.sh:839: get_is_making_recovery_progress: xmlstarlet sel -t -m //pgmap/recovering_keys_per_sec -v . -o ' ' -t -m //pgmap/recovering_bytes_per_sec -v . -o ' ' -t -m //pgmap/recovering_objects_per_sec -v . ../qa/workunits/ceph-helpers.sh:839: get_is_making_recovery_progress: local progress= ../qa/workunits/ceph-helpers.sh:840: get_is_making_recovery_progress: test -n '' ../qa/workunits/ceph-helpers.sh:988: wait_for_clean: (( timer >= 120 )) .../qa/workunits/ceph-helpers.sh:993: wait_for_clean: get_num_active_clean .../qa/workunits/ceph-helpers.sh:863: get_num_active_clean: local 'expression=(' .../qa/workunits/ceph-helpers.sh:864: get_num_active_clean: expression+='contains(.,'\''active'\'') and ' .../qa/workunits/ceph-helpers.sh:865: get_num_active_clean: expression+='contains(.,'\''clean'\'') and ' .../qa/workunits/ceph-helpers.sh:866: get_num_active_clean: expression+='not(contains(.,'\''stale'\''))' .../qa/workunits/ceph-helpers.sh:867: get_num_active_clean: expression+=')' .../qa/workunits/ceph-helpers.sh:871: get_num_active_clean: ceph --format xml pg dump pgs .../qa/workunits/ceph-helpers.sh:872: get_num_active_clean: xmlstarlet sel -t -m '//pg_stat/state[(contains(.,'\''active'\'') and contains(.,'\''clean'\'') and not(contains(.,'\''stale'\'')))]' -v . -n .../qa/workunits/ceph-helpers.sh:873: get_num_active_clean: grep -v '^$' .../qa/workunits/ceph-helpers.sh:873: get_num_active_clean: wc -l ../qa/workunits/ceph-helpers.sh:993: wait_for_clean: cur_active_clean=0 ../qa/workunits/ceph-helpers.sh:994: wait_for_clean: test 0 '!=' 0 ../qa/workunits/ceph-helpers.sh:998: wait_for_clean: sleep 1 .../qa/workunits/ceph-helpers.sh:999: wait_for_clean: expr 54 + 1 ../qa/workunits/ceph-helpers.sh:999: wait_for_clean: timer=55 ../qa/workunits/ceph-helpers.sh:985: wait_for_clean: is_clean .../qa/workunits/ceph-helpers.sh:952: is_clean: get_num_pgs .../qa/workunits/ceph-helpers.sh:898: get_num_pgs: ceph --format xml status .../qa/workunits/ceph-helpers.sh:899: get_num_pgs: xmlstarlet sel -t -m //pgmap/num_pgs -v . ../qa/workunits/ceph-helpers.sh:952: is_clean: num_pgs=4 ../qa/workunits/ceph-helpers.sh:953: is_clean: test 4 '!=' 0 .../qa/workunits/ceph-helpers.sh:954: is_clean: get_num_active_clean .../qa/workunits/ceph-helpers.sh:863: get_num_active_clean: local 'expression=(' .../qa/workunits/ceph-helpers.sh:864: get_num_active_clean: expression+='contains(.,'\''active'\'') and ' .../qa/workunits/ceph-helpers.sh:865: get_num_active_clean: expression+='contains(.,'\''clean'\'') and ' .../qa/workunits/ceph-helpers.sh:866: get_num_active_clean: expression+='not(contains(.,'\''stale'\''))' .../qa/workunits/ceph-helpers.sh:867: get_num_active_clean: expression+=')' .../qa/workunits/ceph-helpers.sh:871: get_num_active_clean: ceph --format xml pg dump pgs .../qa/workunits/ceph-helpers.sh:872: get_num_active_clean: xmlstarlet sel -t -m '//pg_stat/state[(contains(.,'\''active'\'') and contains(.,'\''clean'\'') and not(contains(.,'\''stale'\'')))]' -v . -n .../qa/workunits/ceph-helpers.sh:873: get_num_active_clean: grep -v '^$' .../qa/workunits/ceph-helpers.sh:873: get_num_active_clean: wc -l ../qa/workunits/ceph-helpers.sh:954: is_clean: test 0 = 4 ../qa/workunits/ceph-helpers.sh:954: is_clean: return 1 ../qa/workunits/ceph-helpers.sh:986: wait_for_clean: get_is_making_recovery_progress .../qa/workunits/ceph-helpers.sh:839: get_is_making_recovery_progress: ceph --format xml status .../qa/workunits/ceph-helpers.sh:839: get_is_making_recovery_progress: xmlstarlet sel -t -m //pgmap/recovering_keys_per_sec -v . -o ' ' -t -m //pgmap/recovering_bytes_per_sec -v . -o ' ' -t -m //pgmap/recovering_objects_per_sec -v . ../qa/workunits/ceph-helpers.sh:839: get_is_making_recovery_progress: local progress= ../qa/workunits/ceph-helpers.sh:840: get_is_making_recovery_progress: test -n '' ../qa/workunits/ceph-helpers.sh:988: wait_for_clean: (( timer >= 120 )) .../qa/workunits/ceph-helpers.sh:993: wait_for_clean: get_num_active_clean .../qa/workunits/ceph-helpers.sh:863: get_num_active_clean: local 'expression=(' .../qa/workunits/ceph-helpers.sh:864: get_num_active_clean: expression+='contains(.,'\''active'\'') and ' .../qa/workunits/ceph-helpers.sh:865: get_num_active_clean: expression+='contains(.,'\''clean'\'') and ' .../qa/workunits/ceph-helpers.sh:866: get_num_active_clean: expression+='not(contains(.,'\''stale'\''))' .../qa/workunits/ceph-helpers.sh:867: get_num_active_clean: expression+=')' .../qa/workunits/ceph-helpers.sh:871: get_num_active_clean: ceph --format xml pg dump pgs .../qa/workunits/ceph-helpers.sh:872: get_num_active_clean: xmlstarlet sel -t -m '//pg_stat/state[(contains(.,'\''active'\'') and contains(.,'\''clean'\'') and not(contains(.,'\''stale'\'')))]' -v . -n .../qa/workunits/ceph-helpers.sh:873: get_num_active_clean: grep -v '^$' .../qa/workunits/ceph-helpers.sh:873: get_num_active_clean: wc -l ../qa/workunits/ceph-helpers.sh:993: wait_for_clean: cur_active_clean=0 ../qa/workunits/ceph-helpers.sh:994: wait_for_clean: test 0 '!=' 0 ../qa/workunits/ceph-helpers.sh:998: wait_for_clean: sleep 1 .../qa/workunits/ceph-helpers.sh:999: wait_for_clean: expr 55 + 1 ../qa/workunits/ceph-helpers.sh:999: wait_for_clean: timer=56 ../qa/workunits/ceph-helpers.sh:985: wait_for_clean: is_clean .../qa/workunits/ceph-helpers.sh:952: is_clean: get_num_pgs .../qa/workunits/ceph-helpers.sh:898: get_num_pgs: ceph --format xml status .../qa/workunits/ceph-helpers.sh:899: get_num_pgs: xmlstarlet sel -t -m //pgmap/num_pgs -v . ../qa/workunits/ceph-helpers.sh:952: is_clean: num_pgs=4 ../qa/workunits/ceph-helpers.sh:953: is_clean: test 4 '!=' 0 .../qa/workunits/ceph-helpers.sh:954: is_clean: get_num_active_clean .../qa/workunits/ceph-helpers.sh:863: get_num_active_clean: local 'expression=(' .../qa/workunits/ceph-helpers.sh:864: get_num_active_clean: expression+='contains(.,'\''active'\'') and ' .../qa/workunits/ceph-helpers.sh:865: get_num_active_clean: expression+='contains(.,'\''clean'\'') and ' .../qa/workunits/ceph-helpers.sh:866: get_num_active_clean: expression+='not(contains(.,'\''stale'\''))' .../qa/workunits/ceph-helpers.sh:867: get_num_active_clean: expression+=')' .../qa/workunits/ceph-helpers.sh:871: get_num_active_clean: ceph --format xml pg dump pgs .../qa/workunits/ceph-helpers.sh:872: get_num_active_clean: xmlstarlet sel -t -m '//pg_stat/state[(contains(.,'\''active'\'') and contains(.,'\''clean'\'') and not(contains(.,'\''stale'\'')))]' -v . -n .../qa/workunits/ceph-helpers.sh:873: get_num_active_clean: wc -l .../qa/workunits/ceph-helpers.sh:873: get_num_active_clean: grep -v '^$' ../qa/workunits/ceph-helpers.sh:954: is_clean: test 0 = 4 ../qa/workunits/ceph-helpers.sh:954: is_clean: return 1 ../qa/workunits/ceph-helpers.sh:986: wait_for_clean: get_is_making_recovery_progress .../qa/workunits/ceph-helpers.sh:839: get_is_making_recovery_progress: ceph --format xml status .../qa/workunits/ceph-helpers.sh:839: get_is_making_recovery_progress: xmlstarlet sel -t -m //pgmap/recovering_keys_per_sec -v . -o ' ' -t -m //pgmap/recovering_bytes_per_sec -v . -o ' ' -t -m //pgmap/recovering_objects_per_sec -v . ../qa/workunits/ceph-helpers.sh:839: get_is_making_recovery_progress: local progress= ../qa/workunits/ceph-helpers.sh:840: get_is_making_recovery_progress: test -n '' ../qa/workunits/ceph-helpers.sh:988: wait_for_clean: (( timer >= 120 )) .../qa/workunits/ceph-helpers.sh:993: wait_for_clean: get_num_active_clean .../qa/workunits/ceph-helpers.sh:863: get_num_active_clean: local 'expression=(' .../qa/workunits/ceph-helpers.sh:864: get_num_active_clean: expression+='contains(.,'\''active'\'') and ' .../qa/workunits/ceph-helpers.sh:865: get_num_active_clean: expression+='contains(.,'\''clean'\'') and ' .../qa/workunits/ceph-helpers.sh:866: get_num_active_clean: expression+='not(contains(.,'\''stale'\''))' .../qa/workunits/ceph-helpers.sh:867: get_num_active_clean: expression+=')' .../qa/workunits/ceph-helpers.sh:871: get_num_active_clean: ceph --format xml pg dump pgs .../qa/workunits/ceph-helpers.sh:872: get_num_active_clean: xmlstarlet sel -t -m '//pg_stat/state[(contains(.,'\''active'\'') and contains(.,'\''clean'\'') and not(contains(.,'\''stale'\'')))]' -v . -n .../qa/workunits/ceph-helpers.sh:873: get_num_active_clean: grep -v '^$' .../qa/workunits/ceph-helpers.sh:873: get_num_active_clean: wc -l ../qa/workunits/ceph-helpers.sh:993: wait_for_clean: cur_active_clean=0 ../qa/workunits/ceph-helpers.sh:994: wait_for_clean: test 0 '!=' 0 ../qa/workunits/ceph-helpers.sh:998: wait_for_clean: sleep 1 .../qa/workunits/ceph-helpers.sh:999: wait_for_clean: expr 56 + 1 ../qa/workunits/ceph-helpers.sh:999: wait_for_clean: timer=57 ../qa/workunits/ceph-helpers.sh:985: wait_for_clean: is_clean .../qa/workunits/ceph-helpers.sh:952: is_clean: get_num_pgs .../qa/workunits/ceph-helpers.sh:898: get_num_pgs: ceph --format xml status .../qa/workunits/ceph-helpers.sh:899: get_num_pgs: xmlstarlet sel -t -m //pgmap/num_pgs -v . ../qa/workunits/ceph-helpers.sh:952: is_clean: num_pgs=4 ../qa/workunits/ceph-helpers.sh:953: is_clean: test 4 '!=' 0 .../qa/workunits/ceph-helpers.sh:954: is_clean: get_num_active_clean .../qa/workunits/ceph-helpers.sh:863: get_num_active_clean: local 'expression=(' .../qa/workunits/ceph-helpers.sh:864: get_num_active_clean: expression+='contains(.,'\''active'\'') and ' .../qa/workunits/ceph-helpers.sh:865: get_num_active_clean: expression+='contains(.,'\''clean'\'') and ' .../qa/workunits/ceph-helpers.sh:866: get_num_active_clean: expression+='not(contains(.,'\''stale'\''))' .../qa/workunits/ceph-helpers.sh:867: get_num_active_clean: expression+=')' .../qa/workunits/ceph-helpers.sh:871: get_num_active_clean: ceph --format xml pg dump pgs .../qa/workunits/ceph-helpers.sh:872: get_num_active_clean: xmlstarlet sel -t -m '//pg_stat/state[(contains(.,'\''active'\'') and contains(.,'\''clean'\'') and not(contains(.,'\''stale'\'')))]' -v . -n .../qa/workunits/ceph-helpers.sh:873: get_num_active_clean: grep -v '^$' .../qa/workunits/ceph-helpers.sh:873: get_num_active_clean: wc -l ../qa/workunits/ceph-helpers.sh:954: is_clean: test 0 = 4 ../qa/workunits/ceph-helpers.sh:954: is_clean: return 1 ../qa/workunits/ceph-helpers.sh:986: wait_for_clean: get_is_making_recovery_progress .../qa/workunits/ceph-helpers.sh:839: get_is_making_recovery_progress: ceph --format xml status .../qa/workunits/ceph-helpers.sh:839: get_is_making_recovery_progress: xmlstarlet sel -t -m //pgmap/recovering_keys_per_sec -v . -o ' ' -t -m //pgmap/recovering_bytes_per_sec -v . -o ' ' -t -m //pgmap/recovering_objects_per_sec -v . ../qa/workunits/ceph-helpers.sh:839: get_is_making_recovery_progress: local progress= ../qa/workunits/ceph-helpers.sh:840: get_is_making_recovery_progress: test -n '' ../qa/workunits/ceph-helpers.sh:988: wait_for_clean: (( timer >= 120 )) .../qa/workunits/ceph-helpers.sh:993: wait_for_clean: get_num_active_clean .../qa/workunits/ceph-helpers.sh:863: get_num_active_clean: local 'expression=(' .../qa/workunits/ceph-helpers.sh:864: get_num_active_clean: expression+='contains(.,'\''active'\'') and ' .../qa/workunits/ceph-helpers.sh:865: get_num_active_clean: expression+='contains(.,'\''clean'\'') and ' .../qa/workunits/ceph-helpers.sh:866: get_num_active_clean: expression+='not(contains(.,'\''stale'\''))' .../qa/workunits/ceph-helpers.sh:867: get_num_active_clean: expression+=')' .../qa/workunits/ceph-helpers.sh:871: get_num_active_clean: ceph --format xml pg dump pgs .../qa/workunits/ceph-helpers.sh:872: get_num_active_clean: xmlstarlet sel -t -m '//pg_stat/state[(contains(.,'\''active'\'') and contains(.,'\''clean'\'') and not(contains(.,'\''stale'\'')))]' -v . -n .../qa/workunits/ceph-helpers.sh:873: get_num_active_clean: grep -v '^$' .../qa/workunits/ceph-helpers.sh:873: get_num_active_clean: wc -l ../qa/workunits/ceph-helpers.sh:993: wait_for_clean: cur_active_clean=0 ../qa/workunits/ceph-helpers.sh:994: wait_for_clean: test 0 '!=' 0 ../qa/workunits/ceph-helpers.sh:998: wait_for_clean: sleep 1 .../qa/workunits/ceph-helpers.sh:999: wait_for_clean: expr 57 + 1 ../qa/workunits/ceph-helpers.sh:999: wait_for_clean: timer=58 ../qa/workunits/ceph-helpers.sh:985: wait_for_clean: is_clean .../qa/workunits/ceph-helpers.sh:952: is_clean: get_num_pgs .../qa/workunits/ceph-helpers.sh:898: get_num_pgs: ceph --format xml status .../qa/workunits/ceph-helpers.sh:899: get_num_pgs: xmlstarlet sel -t -m //pgmap/num_pgs -v . ../qa/workunits/ceph-helpers.sh:952: is_clean: num_pgs=4 ../qa/workunits/ceph-helpers.sh:953: is_clean: test 4 '!=' 0 .../qa/workunits/ceph-helpers.sh:954: is_clean: get_num_active_clean .../qa/workunits/ceph-helpers.sh:863: get_num_active_clean: local 'expression=(' .../qa/workunits/ceph-helpers.sh:864: get_num_active_clean: expression+='contains(.,'\''active'\'') and ' .../qa/workunits/ceph-helpers.sh:865: get_num_active_clean: expression+='contains(.,'\''clean'\'') and ' .../qa/workunits/ceph-helpers.sh:866: get_num_active_clean: expression+='not(contains(.,'\''stale'\''))' .../qa/workunits/ceph-helpers.sh:867: get_num_active_clean: expression+=')' .../qa/workunits/ceph-helpers.sh:871: get_num_active_clean: ceph --format xml pg dump pgs .../qa/workunits/ceph-helpers.sh:872: get_num_active_clean: xmlstarlet sel -t -m '//pg_stat/state[(contains(.,'\''active'\'') and contains(.,'\''clean'\'') and not(contains(.,'\''stale'\'')))]' -v . -n .../qa/workunits/ceph-helpers.sh:873: get_num_active_clean: wc -l .../qa/workunits/ceph-helpers.sh:873: get_num_active_clean: grep -v '^$' ../qa/workunits/ceph-helpers.sh:954: is_clean: test 0 = 4 ../qa/workunits/ceph-helpers.sh:954: is_clean: return 1 ../qa/workunits/ceph-helpers.sh:986: wait_for_clean: get_is_making_recovery_progress .../qa/workunits/ceph-helpers.sh:839: get_is_making_recovery_progress: ceph --format xml status .../qa/workunits/ceph-helpers.sh:839: get_is_making_recovery_progress: xmlstarlet sel -t -m //pgmap/recovering_keys_per_sec -v . -o ' ' -t -m //pgmap/recovering_bytes_per_sec -v . -o ' ' -t -m //pgmap/recovering_objects_per_sec -v . ../qa/workunits/ceph-helpers.sh:839: get_is_making_recovery_progress: local progress= ../qa/workunits/ceph-helpers.sh:840: get_is_making_recovery_progress: test -n '' ../qa/workunits/ceph-helpers.sh:988: wait_for_clean: (( timer >= 120 )) .../qa/workunits/ceph-helpers.sh:993: wait_for_clean: get_num_active_clean .../qa/workunits/ceph-helpers.sh:863: get_num_active_clean: local 'expression=(' .../qa/workunits/ceph-helpers.sh:864: get_num_active_clean: expression+='contains(.,'\''active'\'') and ' .../qa/workunits/ceph-helpers.sh:865: get_num_active_clean: expression+='contains(.,'\''clean'\'') and ' .../qa/workunits/ceph-helpers.sh:866: get_num_active_clean: expression+='not(contains(.,'\''stale'\''))' .../qa/workunits/ceph-helpers.sh:867: get_num_active_clean: expression+=')' .../qa/workunits/ceph-helpers.sh:871: get_num_active_clean: ceph --format xml pg dump pgs .../qa/workunits/ceph-helpers.sh:872: get_num_active_clean: xmlstarlet sel -t -m '//pg_stat/state[(contains(.,'\''active'\'') and contains(.,'\''clean'\'') and not(contains(.,'\''stale'\'')))]' -v . -n .../qa/workunits/ceph-helpers.sh:873: get_num_active_clean: grep -v '^$' .../qa/workunits/ceph-helpers.sh:873: get_num_active_clean: wc -l ../qa/workunits/ceph-helpers.sh:993: wait_for_clean: cur_active_clean=0 ../qa/workunits/ceph-helpers.sh:994: wait_for_clean: test 0 '!=' 0 ../qa/workunits/ceph-helpers.sh:998: wait_for_clean: sleep 1 .../qa/workunits/ceph-helpers.sh:999: wait_for_clean: expr 58 + 1 ../qa/workunits/ceph-helpers.sh:999: wait_for_clean: timer=59 ../qa/workunits/ceph-helpers.sh:985: wait_for_clean: is_clean .../qa/workunits/ceph-helpers.sh:952: is_clean: get_num_pgs .../qa/workunits/ceph-helpers.sh:898: get_num_pgs: ceph --format xml status .../qa/workunits/ceph-helpers.sh:899: get_num_pgs: xmlstarlet sel -t -m //pgmap/num_pgs -v . ../qa/workunits/ceph-helpers.sh:952: is_clean: num_pgs=4 ../qa/workunits/ceph-helpers.sh:953: is_clean: test 4 '!=' 0 .../qa/workunits/ceph-helpers.sh:954: is_clean: get_num_active_clean .../qa/workunits/ceph-helpers.sh:863: get_num_active_clean: local 'expression=(' .../qa/workunits/ceph-helpers.sh:864: get_num_active_clean: expression+='contains(.,'\''active'\'') and ' .../qa/workunits/ceph-helpers.sh:865: get_num_active_clean: expression+='contains(.,'\''clean'\'') and ' .../qa/workunits/ceph-helpers.sh:866: get_num_active_clean: expression+='not(contains(.,'\''stale'\''))' .../qa/workunits/ceph-helpers.sh:867: get_num_active_clean: expression+=')' .../qa/workunits/ceph-helpers.sh:871: get_num_active_clean: ceph --format xml pg dump pgs .../qa/workunits/ceph-helpers.sh:872: get_num_active_clean: xmlstarlet sel -t -m '//pg_stat/state[(contains(.,'\''active'\'') and contains(.,'\''clean'\'') and not(contains(.,'\''stale'\'')))]' -v . -n .../qa/workunits/ceph-helpers.sh:873: get_num_active_clean: grep -v '^$' .../qa/workunits/ceph-helpers.sh:873: get_num_active_clean: wc -l ../qa/workunits/ceph-helpers.sh:954: is_clean: test 0 = 4 ../qa/workunits/ceph-helpers.sh:954: is_clean: return 1 ../qa/workunits/ceph-helpers.sh:986: wait_for_clean: get_is_making_recovery_progress .../qa/workunits/ceph-helpers.sh:839: get_is_making_recovery_progress: ceph --format xml status .../qa/workunits/ceph-helpers.sh:839: get_is_making_recovery_progress: xmlstarlet sel -t -m //pgmap/recovering_keys_per_sec -v . -o ' ' -t -m //pgmap/recovering_bytes_per_sec -v . -o ' ' -t -m //pgmap/recovering_objects_per_sec -v . ../qa/workunits/ceph-helpers.sh:839: get_is_making_recovery_progress: local progress= ../qa/workunits/ceph-helpers.sh:840: get_is_making_recovery_progress: test -n '' ../qa/workunits/ceph-helpers.sh:988: wait_for_clean: (( timer >= 120 )) .../qa/workunits/ceph-helpers.sh:993: wait_for_clean: get_num_active_clean .../qa/workunits/ceph-helpers.sh:863: get_num_active_clean: local 'expression=(' .../qa/workunits/ceph-helpers.sh:864: get_num_active_clean: expression+='contains(.,'\''active'\'') and ' .../qa/workunits/ceph-helpers.sh:865: get_num_active_clean: expression+='contains(.,'\''clean'\'') and ' .../qa/workunits/ceph-helpers.sh:866: get_num_active_clean: expression+='not(contains(.,'\''stale'\''))' .../qa/workunits/ceph-helpers.sh:867: get_num_active_clean: expression+=')' .../qa/workunits/ceph-helpers.sh:871: get_num_active_clean: ceph --format xml pg dump pgs .../qa/workunits/ceph-helpers.sh:872: get_num_active_clean: xmlstarlet sel -t -m '//pg_stat/state[(contains(.,'\''active'\'') and contains(.,'\''clean'\'') and not(contains(.,'\''stale'\'')))]' -v . -n .../qa/workunits/ceph-helpers.sh:873: get_num_active_clean: grep -v '^$' .../qa/workunits/ceph-helpers.sh:873: get_num_active_clean: wc -l ../qa/workunits/ceph-helpers.sh:993: wait_for_clean: cur_active_clean=0 ../qa/workunits/ceph-helpers.sh:994: wait_for_clean: test 0 '!=' 0 ../qa/workunits/ceph-helpers.sh:998: wait_for_clean: sleep 1 .../qa/workunits/ceph-helpers.sh:999: wait_for_clean: expr 59 + 1 ../qa/workunits/ceph-helpers.sh:999: wait_for_clean: timer=60 ../qa/workunits/ceph-helpers.sh:985: wait_for_clean: is_clean .../qa/workunits/ceph-helpers.sh:952: is_clean: get_num_pgs .../qa/workunits/ceph-helpers.sh:898: get_num_pgs: ceph --format xml status .../qa/workunits/ceph-helpers.sh:899: get_num_pgs: xmlstarlet sel -t -m //pgmap/num_pgs -v . ../qa/workunits/ceph-helpers.sh:952: is_clean: num_pgs=4 ../qa/workunits/ceph-helpers.sh:953: is_clean: test 4 '!=' 0 .../qa/workunits/ceph-helpers.sh:954: is_clean: get_num_active_clean .../qa/workunits/ceph-helpers.sh:863: get_num_active_clean: local 'expression=(' .../qa/workunits/ceph-helpers.sh:864: get_num_active_clean: expression+='contains(.,'\''active'\'') and ' .../qa/workunits/ceph-helpers.sh:865: get_num_active_clean: expression+='contains(.,'\''clean'\'') and ' .../qa/workunits/ceph-helpers.sh:866: get_num_active_clean: expression+='not(contains(.,'\''stale'\''))' .../qa/workunits/ceph-helpers.sh:867: get_num_active_clean: expression+=')' .../qa/workunits/ceph-helpers.sh:871: get_num_active_clean: ceph --format xml pg dump pgs .../qa/workunits/ceph-helpers.sh:872: get_num_active_clean: xmlstarlet sel -t -m '//pg_stat/state[(contains(.,'\''active'\'') and contains(.,'\''clean'\'') and not(contains(.,'\''stale'\'')))]' -v . -n .../qa/workunits/ceph-helpers.sh:873: get_num_active_clean: wc -l .../qa/workunits/ceph-helpers.sh:873: get_num_active_clean: grep -v '^$' ../qa/workunits/ceph-helpers.sh:954: is_clean: test 0 = 4 ../qa/workunits/ceph-helpers.sh:954: is_clean: return 1 ../qa/workunits/ceph-helpers.sh:986: wait_for_clean: get_is_making_recovery_progress .../qa/workunits/ceph-helpers.sh:839: get_is_making_recovery_progress: ceph --format xml status .../qa/workunits/ceph-helpers.sh:839: get_is_making_recovery_progress: xmlstarlet sel -t -m //pgmap/recovering_keys_per_sec -v . -o ' ' -t -m //pgmap/recovering_bytes_per_sec -v . -o ' ' -t -m //pgmap/recovering_objects_per_sec -v . ../qa/workunits/ceph-helpers.sh:839: get_is_making_recovery_progress: local progress= ../qa/workunits/ceph-helpers.sh:840: get_is_making_recovery_progress: test -n '' ../qa/workunits/ceph-helpers.sh:988: wait_for_clean: (( timer >= 120 )) .../qa/workunits/ceph-helpers.sh:993: wait_for_clean: get_num_active_clean .../qa/workunits/ceph-helpers.sh:863: get_num_active_clean: local 'expression=(' .../qa/workunits/ceph-helpers.sh:864: get_num_active_clean: expression+='contains(.,'\''active'\'') and ' .../qa/workunits/ceph-helpers.sh:865: get_num_active_clean: expression+='contains(.,'\''clean'\'') and ' .../qa/workunits/ceph-helpers.sh:866: get_num_active_clean: expression+='not(contains(.,'\''stale'\''))' .../qa/workunits/ceph-helpers.sh:867: get_num_active_clean: expression+=')' .../qa/workunits/ceph-helpers.sh:871: get_num_active_clean: ceph --format xml pg dump pgs .../qa/workunits/ceph-helpers.sh:872: get_num_active_clean: xmlstarlet sel -t -m '//pg_stat/state[(contains(.,'\''active'\'') and contains(.,'\''clean'\'') and not(contains(.,'\''stale'\'')))]' -v . -n .../qa/workunits/ceph-helpers.sh:873: get_num_active_clean: wc -l .../qa/workunits/ceph-helpers.sh:873: get_num_active_clean: grep -v '^$' ../qa/workunits/ceph-helpers.sh:993: wait_for_clean: cur_active_clean=0 ../qa/workunits/ceph-helpers.sh:994: wait_for_clean: test 0 '!=' 0 ../qa/workunits/ceph-helpers.sh:998: wait_for_clean: sleep 1 .../qa/workunits/ceph-helpers.sh:999: wait_for_clean: expr 60 + 1 ../qa/workunits/ceph-helpers.sh:999: wait_for_clean: timer=61 ../qa/workunits/ceph-helpers.sh:985: wait_for_clean: is_clean .../qa/workunits/ceph-helpers.sh:952: is_clean: get_num_pgs .../qa/workunits/ceph-helpers.sh:898: get_num_pgs: ceph --format xml status .../qa/workunits/ceph-helpers.sh:899: get_num_pgs: xmlstarlet sel -t -m //pgmap/num_pgs -v . ../qa/workunits/ceph-helpers.sh:952: is_clean: num_pgs=4 ../qa/workunits/ceph-helpers.sh:953: is_clean: test 4 '!=' 0 .../qa/workunits/ceph-helpers.sh:954: is_clean: get_num_active_clean .../qa/workunits/ceph-helpers.sh:863: get_num_active_clean: local 'expression=(' .../qa/workunits/ceph-helpers.sh:864: get_num_active_clean: expression+='contains(.,'\''active'\'') and ' .../qa/workunits/ceph-helpers.sh:865: get_num_active_clean: expression+='contains(.,'\''clean'\'') and ' .../qa/workunits/ceph-helpers.sh:866: get_num_active_clean: expression+='not(contains(.,'\''stale'\''))' .../qa/workunits/ceph-helpers.sh:867: get_num_active_clean: expression+=')' .../qa/workunits/ceph-helpers.sh:871: get_num_active_clean: ceph --format xml pg dump pgs .../qa/workunits/ceph-helpers.sh:872: get_num_active_clean: xmlstarlet sel -t -m '//pg_stat/state[(contains(.,'\''active'\'') and contains(.,'\''clean'\'') and not(contains(.,'\''stale'\'')))]' -v . -n .../qa/workunits/ceph-helpers.sh:873: get_num_active_clean: grep -v '^$' .../qa/workunits/ceph-helpers.sh:873: get_num_active_clean: wc -l ../qa/workunits/ceph-helpers.sh:954: is_clean: test 0 = 4 ../qa/workunits/ceph-helpers.sh:954: is_clean: return 1 ../qa/workunits/ceph-helpers.sh:986: wait_for_clean: get_is_making_recovery_progress .../qa/workunits/ceph-helpers.sh:839: get_is_making_recovery_progress: ceph --format xml status .../qa/workunits/ceph-helpers.sh:839: get_is_making_recovery_progress: xmlstarlet sel -t -m //pgmap/recovering_keys_per_sec -v . -o ' ' -t -m //pgmap/recovering_bytes_per_sec -v . -o ' ' -t -m //pgmap/recovering_objects_per_sec -v . ../qa/workunits/ceph-helpers.sh:839: get_is_making_recovery_progress: local progress= ../qa/workunits/ceph-helpers.sh:840: get_is_making_recovery_progress: test -n '' ../qa/workunits/ceph-helpers.sh:988: wait_for_clean: (( timer >= 120 )) .../qa/workunits/ceph-helpers.sh:993: wait_for_clean: get_num_active_clean .../qa/workunits/ceph-helpers.sh:863: get_num_active_clean: local 'expression=(' .../qa/workunits/ceph-helpers.sh:864: get_num_active_clean: expression+='contains(.,'\''active'\'') and ' .../qa/workunits/ceph-helpers.sh:865: get_num_active_clean: expression+='contains(.,'\''clean'\'') and ' .../qa/workunits/ceph-helpers.sh:866: get_num_active_clean: expression+='not(contains(.,'\''stale'\''))' .../qa/workunits/ceph-helpers.sh:867: get_num_active_clean: expression+=')' .../qa/workunits/ceph-helpers.sh:871: get_num_active_clean: ceph --format xml pg dump pgs .../qa/workunits/ceph-helpers.sh:872: get_num_active_clean: xmlstarlet sel -t -m '//pg_stat/state[(contains(.,'\''active'\'') and contains(.,'\''clean'\'') and not(contains(.,'\''stale'\'')))]' -v . -n .../qa/workunits/ceph-helpers.sh:873: get_num_active_clean: grep -v '^$' .../qa/workunits/ceph-helpers.sh:873: get_num_active_clean: wc -l ../qa/workunits/ceph-helpers.sh:993: wait_for_clean: cur_active_clean=0 ../qa/workunits/ceph-helpers.sh:994: wait_for_clean: test 0 '!=' 0 ../qa/workunits/ceph-helpers.sh:998: wait_for_clean: sleep 1 .../qa/workunits/ceph-helpers.sh:999: wait_for_clean: expr 61 + 1 ../qa/workunits/ceph-helpers.sh:999: wait_for_clean: timer=62 ../qa/workunits/ceph-helpers.sh:985: wait_for_clean: is_clean .../qa/workunits/ceph-helpers.sh:952: is_clean: get_num_pgs .../qa/workunits/ceph-helpers.sh:898: get_num_pgs: ceph --format xml status .../qa/workunits/ceph-helpers.sh:899: get_num_pgs: xmlstarlet sel -t -m //pgmap/num_pgs -v . ../qa/workunits/ceph-helpers.sh:952: is_clean: num_pgs=4 ../qa/workunits/ceph-helpers.sh:953: is_clean: test 4 '!=' 0 .../qa/workunits/ceph-helpers.sh:954: is_clean: get_num_active_clean .../qa/workunits/ceph-helpers.sh:863: get_num_active_clean: local 'expression=(' .../qa/workunits/ceph-helpers.sh:864: get_num_active_clean: expression+='contains(.,'\''active'\'') and ' .../qa/workunits/ceph-helpers.sh:865: get_num_active_clean: expression+='contains(.,'\''clean'\'') and ' .../qa/workunits/ceph-helpers.sh:866: get_num_active_clean: expression+='not(contains(.,'\''stale'\''))' .../qa/workunits/ceph-helpers.sh:867: get_num_active_clean: expression+=')' .../qa/workunits/ceph-helpers.sh:871: get_num_active_clean: ceph --format xml pg dump pgs .../qa/workunits/ceph-helpers.sh:873: get_num_active_clean: grep -v '^$' .../qa/workunits/ceph-helpers.sh:872: get_num_active_clean: xmlstarlet sel -t -m '//pg_stat/state[(contains(.,'\''active'\'') and contains(.,'\''clean'\'') and not(contains(.,'\''stale'\'')))]' -v . -n .../qa/workunits/ceph-helpers.sh:873: get_num_active_clean: wc -l ../qa/workunits/ceph-helpers.sh:954: is_clean: test 0 = 4 ../qa/workunits/ceph-helpers.sh:954: is_clean: return 1 ../qa/workunits/ceph-helpers.sh:986: wait_for_clean: get_is_making_recovery_progress .../qa/workunits/ceph-helpers.sh:839: get_is_making_recovery_progress: ceph --format xml status .../qa/workunits/ceph-helpers.sh:839: get_is_making_recovery_progress: xmlstarlet sel -t -m //pgmap/recovering_keys_per_sec -v . -o ' ' -t -m //pgmap/recovering_bytes_per_sec -v . -o ' ' -t -m //pgmap/recovering_objects_per_sec -v . ../qa/workunits/ceph-helpers.sh:839: get_is_making_recovery_progress: local progress= ../qa/workunits/ceph-helpers.sh:840: get_is_making_recovery_progress: test -n '' ../qa/workunits/ceph-helpers.sh:988: wait_for_clean: (( timer >= 120 )) .../qa/workunits/ceph-helpers.sh:993: wait_for_clean: get_num_active_clean .../qa/workunits/ceph-helpers.sh:863: get_num_active_clean: local 'expression=(' .../qa/workunits/ceph-helpers.sh:864: get_num_active_clean: expression+='contains(.,'\''active'\'') and ' .../qa/workunits/ceph-helpers.sh:865: get_num_active_clean: expression+='contains(.,'\''clean'\'') and ' .../qa/workunits/ceph-helpers.sh:866: get_num_active_clean: expression+='not(contains(.,'\''stale'\''))' .../qa/workunits/ceph-helpers.sh:867: get_num_active_clean: expression+=')' .../qa/workunits/ceph-helpers.sh:871: get_num_active_clean: ceph --format xml pg dump pgs .../qa/workunits/ceph-helpers.sh:872: get_num_active_clean: xmlstarlet sel -t -m '//pg_stat/state[(contains(.,'\''active'\'') and contains(.,'\''clean'\'') and not(contains(.,'\''stale'\'')))]' -v . -n .../qa/workunits/ceph-helpers.sh:873: get_num_active_clean: grep -v '^$' .../qa/workunits/ceph-helpers.sh:873: get_num_active_clean: wc -l ../qa/workunits/ceph-helpers.sh:993: wait_for_clean: cur_active_clean=0 ../qa/workunits/ceph-helpers.sh:994: wait_for_clean: test 0 '!=' 0 ../qa/workunits/ceph-helpers.sh:998: wait_for_clean: sleep 1 .../qa/workunits/ceph-helpers.sh:999: wait_for_clean: expr 62 + 1 ../qa/workunits/ceph-helpers.sh:999: wait_for_clean: timer=63 ../qa/workunits/ceph-helpers.sh:985: wait_for_clean: is_clean .../qa/workunits/ceph-helpers.sh:952: is_clean: get_num_pgs .../qa/workunits/ceph-helpers.sh:898: get_num_pgs: ceph --format xml status .../qa/workunits/ceph-helpers.sh:899: get_num_pgs: xmlstarlet sel -t -m //pgmap/num_pgs -v . ../qa/workunits/ceph-helpers.sh:952: is_clean: num_pgs=4 ../qa/workunits/ceph-helpers.sh:953: is_clean: test 4 '!=' 0 .../qa/workunits/ceph-helpers.sh:954: is_clean: get_num_active_clean .../qa/workunits/ceph-helpers.sh:863: get_num_active_clean: local 'expression=(' .../qa/workunits/ceph-helpers.sh:864: get_num_active_clean: expression+='contains(.,'\''active'\'') and ' .../qa/workunits/ceph-helpers.sh:865: get_num_active_clean: expression+='contains(.,'\''clean'\'') and ' .../qa/workunits/ceph-helpers.sh:866: get_num_active_clean: expression+='not(contains(.,'\''stale'\''))' .../qa/workunits/ceph-helpers.sh:867: get_num_active_clean: expression+=')' .../qa/workunits/ceph-helpers.sh:871: get_num_active_clean: ceph --format xml pg dump pgs .../qa/workunits/ceph-helpers.sh:872: get_num_active_clean: xmlstarlet sel -t -m '//pg_stat/state[(contains(.,'\''active'\'') and contains(.,'\''clean'\'') and not(contains(.,'\''stale'\'')))]' -v . -n .../qa/workunits/ceph-helpers.sh:873: get_num_active_clean: grep -v '^$' .../qa/workunits/ceph-helpers.sh:873: get_num_active_clean: wc -l ../qa/workunits/ceph-helpers.sh:954: is_clean: test 0 = 4 ../qa/workunits/ceph-helpers.sh:954: is_clean: return 1 ../qa/workunits/ceph-helpers.sh:986: wait_for_clean: get_is_making_recovery_progress .../qa/workunits/ceph-helpers.sh:839: get_is_making_recovery_progress: ceph --format xml status .../qa/workunits/ceph-helpers.sh:839: get_is_making_recovery_progress: xmlstarlet sel -t -m //pgmap/recovering_keys_per_sec -v . -o ' ' -t -m //pgmap/recovering_bytes_per_sec -v . -o ' ' -t -m //pgmap/recovering_objects_per_sec -v . ../qa/workunits/ceph-helpers.sh:839: get_is_making_recovery_progress: local progress= ../qa/workunits/ceph-helpers.sh:840: get_is_making_recovery_progress: test -n '' ../qa/workunits/ceph-helpers.sh:988: wait_for_clean: (( timer >= 120 )) .../qa/workunits/ceph-helpers.sh:993: wait_for_clean: get_num_active_clean .../qa/workunits/ceph-helpers.sh:863: get_num_active_clean: local 'expression=(' .../qa/workunits/ceph-helpers.sh:864: get_num_active_clean: expression+='contains(.,'\''active'\'') and ' .../qa/workunits/ceph-helpers.sh:865: get_num_active_clean: expression+='contains(.,'\''clean'\'') and ' .../qa/workunits/ceph-helpers.sh:866: get_num_active_clean: expression+='not(contains(.,'\''stale'\''))' .../qa/workunits/ceph-helpers.sh:867: get_num_active_clean: expression+=')' .../qa/workunits/ceph-helpers.sh:871: get_num_active_clean: ceph --format xml pg dump pgs .../qa/workunits/ceph-helpers.sh:872: get_num_active_clean: xmlstarlet sel -t -m '//pg_stat/state[(contains(.,'\''active'\'') and contains(.,'\''clean'\'') and not(contains(.,'\''stale'\'')))]' -v . -n .../qa/workunits/ceph-helpers.sh:873: get_num_active_clean: grep -v '^$' .../qa/workunits/ceph-helpers.sh:873: get_num_active_clean: wc -l ../qa/workunits/ceph-helpers.sh:993: wait_for_clean: cur_active_clean=0 ../qa/workunits/ceph-helpers.sh:994: wait_for_clean: test 0 '!=' 0 ../qa/workunits/ceph-helpers.sh:998: wait_for_clean: sleep 1 .../qa/workunits/ceph-helpers.sh:999: wait_for_clean: expr 63 + 1 ../qa/workunits/ceph-helpers.sh:999: wait_for_clean: timer=64 ../qa/workunits/ceph-helpers.sh:985: wait_for_clean: is_clean .../qa/workunits/ceph-helpers.sh:952: is_clean: get_num_pgs .../qa/workunits/ceph-helpers.sh:898: get_num_pgs: ceph --format xml status .../qa/workunits/ceph-helpers.sh:899: get_num_pgs: xmlstarlet sel -t -m //pgmap/num_pgs -v . ../qa/workunits/ceph-helpers.sh:952: is_clean: num_pgs=4 ../qa/workunits/ceph-helpers.sh:953: is_clean: test 4 '!=' 0 .../qa/workunits/ceph-helpers.sh:954: is_clean: get_num_active_clean .../qa/workunits/ceph-helpers.sh:863: get_num_active_clean: local 'expression=(' .../qa/workunits/ceph-helpers.sh:864: get_num_active_clean: expression+='contains(.,'\''active'\'') and ' .../qa/workunits/ceph-helpers.sh:865: get_num_active_clean: expression+='contains(.,'\''clean'\'') and ' .../qa/workunits/ceph-helpers.sh:866: get_num_active_clean: expression+='not(contains(.,'\''stale'\''))' .../qa/workunits/ceph-helpers.sh:867: get_num_active_clean: expression+=')' .../qa/workunits/ceph-helpers.sh:871: get_num_active_clean: ceph --format xml pg dump pgs .../qa/workunits/ceph-helpers.sh:872: get_num_active_clean: xmlstarlet sel -t -m '//pg_stat/state[(contains(.,'\''active'\'') and contains(.,'\''clean'\'') and not(contains(.,'\''stale'\'')))]' -v . -n .../qa/workunits/ceph-helpers.sh:873: get_num_active_clean: grep -v '^$' .../qa/workunits/ceph-helpers.sh:873: get_num_active_clean: wc -l ../qa/workunits/ceph-helpers.sh:954: is_clean: test 0 = 4 ../qa/workunits/ceph-helpers.sh:954: is_clean: return 1 ../qa/workunits/ceph-helpers.sh:986: wait_for_clean: get_is_making_recovery_progress .../qa/workunits/ceph-helpers.sh:839: get_is_making_recovery_progress: ceph --format xml status .../qa/workunits/ceph-helpers.sh:839: get_is_making_recovery_progress: xmlstarlet sel -t -m //pgmap/recovering_keys_per_sec -v . -o ' ' -t -m //pgmap/recovering_bytes_per_sec -v . -o ' ' -t -m //pgmap/recovering_objects_per_sec -v . ../qa/workunits/ceph-helpers.sh:839: get_is_making_recovery_progress: local progress= ../qa/workunits/ceph-helpers.sh:840: get_is_making_recovery_progress: test -n '' ../qa/workunits/ceph-helpers.sh:988: wait_for_clean: (( timer >= 120 )) .../qa/workunits/ceph-helpers.sh:993: wait_for_clean: get_num_active_clean .../qa/workunits/ceph-helpers.sh:863: get_num_active_clean: local 'expression=(' .../qa/workunits/ceph-helpers.sh:864: get_num_active_clean: expression+='contains(.,'\''active'\'') and ' .../qa/workunits/ceph-helpers.sh:865: get_num_active_clean: expression+='contains(.,'\''clean'\'') and ' .../qa/workunits/ceph-helpers.sh:866: get_num_active_clean: expression+='not(contains(.,'\''stale'\''))' .../qa/workunits/ceph-helpers.sh:867: get_num_active_clean: expression+=')' .../qa/workunits/ceph-helpers.sh:871: get_num_active_clean: ceph --format xml pg dump pgs .../qa/workunits/ceph-helpers.sh:872: get_num_active_clean: xmlstarlet sel -t -m '//pg_stat/state[(contains(.,'\''active'\'') and contains(.,'\''clean'\'') and not(contains(.,'\''stale'\'')))]' -v . -n .../qa/workunits/ceph-helpers.sh:873: get_num_active_clean: wc -l .../qa/workunits/ceph-helpers.sh:873: get_num_active_clean: grep -v '^$' ../qa/workunits/ceph-helpers.sh:993: wait_for_clean: cur_active_clean=0 ../qa/workunits/ceph-helpers.sh:994: wait_for_clean: test 0 '!=' 0 ../qa/workunits/ceph-helpers.sh:998: wait_for_clean: sleep 1 .../qa/workunits/ceph-helpers.sh:999: wait_for_clean: expr 64 + 1 ../qa/workunits/ceph-helpers.sh:999: wait_for_clean: timer=65 ../qa/workunits/ceph-helpers.sh:985: wait_for_clean: is_clean .../qa/workunits/ceph-helpers.sh:952: is_clean: get_num_pgs .../qa/workunits/ceph-helpers.sh:898: get_num_pgs: ceph --format xml status .../qa/workunits/ceph-helpers.sh:899: get_num_pgs: xmlstarlet sel -t -m //pgmap/num_pgs -v . ../qa/workunits/ceph-helpers.sh:952: is_clean: num_pgs=4 ../qa/workunits/ceph-helpers.sh:953: is_clean: test 4 '!=' 0 .../qa/workunits/ceph-helpers.sh:954: is_clean: get_num_active_clean .../qa/workunits/ceph-helpers.sh:863: get_num_active_clean: local 'expression=(' .../qa/workunits/ceph-helpers.sh:864: get_num_active_clean: expression+='contains(.,'\''active'\'') and ' .../qa/workunits/ceph-helpers.sh:865: get_num_active_clean: expression+='contains(.,'\''clean'\'') and ' .../qa/workunits/ceph-helpers.sh:866: get_num_active_clean: expression+='not(contains(.,'\''stale'\''))' .../qa/workunits/ceph-helpers.sh:867: get_num_active_clean: expression+=')' .../qa/workunits/ceph-helpers.sh:871: get_num_active_clean: ceph --format xml pg dump pgs .../qa/workunits/ceph-helpers.sh:872: get_num_active_clean: xmlstarlet sel -t -m '//pg_stat/state[(contains(.,'\''active'\'') and contains(.,'\''clean'\'') and not(contains(.,'\''stale'\'')))]' -v . -n .../qa/workunits/ceph-helpers.sh:873: get_num_active_clean: grep -v '^$' .../qa/workunits/ceph-helpers.sh:873: get_num_active_clean: wc -l ../qa/workunits/ceph-helpers.sh:954: is_clean: test 0 = 4 ../qa/workunits/ceph-helpers.sh:954: is_clean: return 1 ../qa/workunits/ceph-helpers.sh:986: wait_for_clean: get_is_making_recovery_progress .../qa/workunits/ceph-helpers.sh:839: get_is_making_recovery_progress: ceph --format xml status .../qa/workunits/ceph-helpers.sh:839: get_is_making_recovery_progress: xmlstarlet sel -t -m //pgmap/recovering_keys_per_sec -v . -o ' ' -t -m //pgmap/recovering_bytes_per_sec -v . -o ' ' -t -m //pgmap/recovering_objects_per_sec -v . ../qa/workunits/ceph-helpers.sh:839: get_is_making_recovery_progress: local progress= ../qa/workunits/ceph-helpers.sh:840: get_is_making_recovery_progress: test -n '' ../qa/workunits/ceph-helpers.sh:988: wait_for_clean: (( timer >= 120 )) .../qa/workunits/ceph-helpers.sh:993: wait_for_clean: get_num_active_clean .../qa/workunits/ceph-helpers.sh:863: get_num_active_clean: local 'expression=(' .../qa/workunits/ceph-helpers.sh:864: get_num_active_clean: expression+='contains(.,'\''active'\'') and ' .../qa/workunits/ceph-helpers.sh:865: get_num_active_clean: expression+='contains(.,'\''clean'\'') and ' .../qa/workunits/ceph-helpers.sh:866: get_num_active_clean: expression+='not(contains(.,'\''stale'\''))' .../qa/workunits/ceph-helpers.sh:867: get_num_active_clean: expression+=')' .../qa/workunits/ceph-helpers.sh:871: get_num_active_clean: ceph --format xml pg dump pgs .../qa/workunits/ceph-helpers.sh:872: get_num_active_clean: xmlstarlet sel -t -m '//pg_stat/state[(contains(.,'\''active'\'') and contains(.,'\''clean'\'') and not(contains(.,'\''stale'\'')))]' -v . -n .../qa/workunits/ceph-helpers.sh:873: get_num_active_clean: grep -v '^$' .../qa/workunits/ceph-helpers.sh:873: get_num_active_clean: wc -l ../qa/workunits/ceph-helpers.sh:993: wait_for_clean: cur_active_clean=0 ../qa/workunits/ceph-helpers.sh:994: wait_for_clean: test 0 '!=' 0 ../qa/workunits/ceph-helpers.sh:998: wait_for_clean: sleep 1 .../qa/workunits/ceph-helpers.sh:999: wait_for_clean: expr 65 + 1 ../qa/workunits/ceph-helpers.sh:999: wait_for_clean: timer=66 ../qa/workunits/ceph-helpers.sh:985: wait_for_clean: is_clean .../qa/workunits/ceph-helpers.sh:952: is_clean: get_num_pgs .../qa/workunits/ceph-helpers.sh:898: get_num_pgs: ceph --format xml status .../qa/workunits/ceph-helpers.sh:899: get_num_pgs: xmlstarlet sel -t -m //pgmap/num_pgs -v . ../qa/workunits/ceph-helpers.sh:952: is_clean: num_pgs=4 ../qa/workunits/ceph-helpers.sh:953: is_clean: test 4 '!=' 0 .../qa/workunits/ceph-helpers.sh:954: is_clean: get_num_active_clean .../qa/workunits/ceph-helpers.sh:863: get_num_active_clean: local 'expression=(' .../qa/workunits/ceph-helpers.sh:864: get_num_active_clean: expression+='contains(.,'\''active'\'') and ' .../qa/workunits/ceph-helpers.sh:865: get_num_active_clean: expression+='contains(.,'\''clean'\'') and ' .../qa/workunits/ceph-helpers.sh:866: get_num_active_clean: expression+='not(contains(.,'\''stale'\''))' .../qa/workunits/ceph-helpers.sh:867: get_num_active_clean: expression+=')' .../qa/workunits/ceph-helpers.sh:871: get_num_active_clean: ceph --format xml pg dump pgs .../qa/workunits/ceph-helpers.sh:872: get_num_active_clean: xmlstarlet sel -t -m '//pg_stat/state[(contains(.,'\''active'\'') and contains(.,'\''clean'\'') and not(contains(.,'\''stale'\'')))]' -v . -n .../qa/workunits/ceph-helpers.sh:873: get_num_active_clean: grep -v '^$' .../qa/workunits/ceph-helpers.sh:873: get_num_active_clean: wc -l ../qa/workunits/ceph-helpers.sh:954: is_clean: test 0 = 4 ../qa/workunits/ceph-helpers.sh:954: is_clean: return 1 ../qa/workunits/ceph-helpers.sh:986: wait_for_clean: get_is_making_recovery_progress .../qa/workunits/ceph-helpers.sh:839: get_is_making_recovery_progress: ceph --format xml status .../qa/workunits/ceph-helpers.sh:839: get_is_making_recovery_progress: xmlstarlet sel -t -m //pgmap/recovering_keys_per_sec -v . -o ' ' -t -m //pgmap/recovering_bytes_per_sec -v . -o ' ' -t -m //pgmap/recovering_objects_per_sec -v . ../qa/workunits/ceph-helpers.sh:839: get_is_making_recovery_progress: local progress= ../qa/workunits/ceph-helpers.sh:840: get_is_making_recovery_progress: test -n '' ../qa/workunits/ceph-helpers.sh:988: wait_for_clean: (( timer >= 120 )) .../qa/workunits/ceph-helpers.sh:993: wait_for_clean: get_num_active_clean .../qa/workunits/ceph-helpers.sh:863: get_num_active_clean: local 'expression=(' .../qa/workunits/ceph-helpers.sh:864: get_num_active_clean: expression+='contains(.,'\''active'\'') and ' .../qa/workunits/ceph-helpers.sh:865: get_num_active_clean: expression+='contains(.,'\''clean'\'') and ' .../qa/workunits/ceph-helpers.sh:866: get_num_active_clean: expression+='not(contains(.,'\''stale'\''))' .../qa/workunits/ceph-helpers.sh:867: get_num_active_clean: expression+=')' .../qa/workunits/ceph-helpers.sh:871: get_num_active_clean: ceph --format xml pg dump pgs .../qa/workunits/ceph-helpers.sh:872: get_num_active_clean: xmlstarlet sel -t -m '//pg_stat/state[(contains(.,'\''active'\'') and contains(.,'\''clean'\'') and not(contains(.,'\''stale'\'')))]' -v . -n .../qa/workunits/ceph-helpers.sh:873: get_num_active_clean: grep -v '^$' .../qa/workunits/ceph-helpers.sh:873: get_num_active_clean: wc -l ../qa/workunits/ceph-helpers.sh:993: wait_for_clean: cur_active_clean=0 ../qa/workunits/ceph-helpers.sh:994: wait_for_clean: test 0 '!=' 0 ../qa/workunits/ceph-helpers.sh:998: wait_for_clean: sleep 1 .../qa/workunits/ceph-helpers.sh:999: wait_for_clean: expr 66 + 1 ../qa/workunits/ceph-helpers.sh:999: wait_for_clean: timer=67 ../qa/workunits/ceph-helpers.sh:985: wait_for_clean: is_clean .../qa/workunits/ceph-helpers.sh:952: is_clean: get_num_pgs .../qa/workunits/ceph-helpers.sh:898: get_num_pgs: ceph --format xml status .../qa/workunits/ceph-helpers.sh:899: get_num_pgs: xmlstarlet sel -t -m //pgmap/num_pgs -v . ../qa/workunits/ceph-helpers.sh:952: is_clean: num_pgs=4 ../qa/workunits/ceph-helpers.sh:953: is_clean: test 4 '!=' 0 .../qa/workunits/ceph-helpers.sh:954: is_clean: get_num_active_clean .../qa/workunits/ceph-helpers.sh:863: get_num_active_clean: local 'expression=(' .../qa/workunits/ceph-helpers.sh:864: get_num_active_clean: expression+='contains(.,'\''active'\'') and ' .../qa/workunits/ceph-helpers.sh:865: get_num_active_clean: expression+='contains(.,'\''clean'\'') and ' .../qa/workunits/ceph-helpers.sh:866: get_num_active_clean: expression+='not(contains(.,'\''stale'\''))' .../qa/workunits/ceph-helpers.sh:867: get_num_active_clean: expression+=')' .../qa/workunits/ceph-helpers.sh:871: get_num_active_clean: ceph --format xml pg dump pgs .../qa/workunits/ceph-helpers.sh:872: get_num_active_clean: xmlstarlet sel -t -m '//pg_stat/state[(contains(.,'\''active'\'') and contains(.,'\''clean'\'') and not(contains(.,'\''stale'\'')))]' -v . -n .../qa/workunits/ceph-helpers.sh:873: get_num_active_clean: wc -l .../qa/workunits/ceph-helpers.sh:873: get_num_active_clean: grep -v '^$' ../qa/workunits/ceph-helpers.sh:954: is_clean: test 0 = 4 ../qa/workunits/ceph-helpers.sh:954: is_clean: return 1 ../qa/workunits/ceph-helpers.sh:986: wait_for_clean: get_is_making_recovery_progress .../qa/workunits/ceph-helpers.sh:839: get_is_making_recovery_progress: ceph --format xml status .../qa/workunits/ceph-helpers.sh:839: get_is_making_recovery_progress: xmlstarlet sel -t -m //pgmap/recovering_keys_per_sec -v . -o ' ' -t -m //pgmap/recovering_bytes_per_sec -v . -o ' ' -t -m //pgmap/recovering_objects_per_sec -v . ../qa/workunits/ceph-helpers.sh:839: get_is_making_recovery_progress: local progress= ../qa/workunits/ceph-helpers.sh:840: get_is_making_recovery_progress: test -n '' ../qa/workunits/ceph-helpers.sh:988: wait_for_clean: (( timer >= 120 )) .../qa/workunits/ceph-helpers.sh:993: wait_for_clean: get_num_active_clean .../qa/workunits/ceph-helpers.sh:863: get_num_active_clean: local 'expression=(' .../qa/workunits/ceph-helpers.sh:864: get_num_active_clean: expression+='contains(.,'\''active'\'') and ' .../qa/workunits/ceph-helpers.sh:865: get_num_active_clean: expression+='contains(.,'\''clean'\'') and ' .../qa/workunits/ceph-helpers.sh:866: get_num_active_clean: expression+='not(contains(.,'\''stale'\''))' .../qa/workunits/ceph-helpers.sh:867: get_num_active_clean: expression+=')' .../qa/workunits/ceph-helpers.sh:871: get_num_active_clean: ceph --format xml pg dump pgs .../qa/workunits/ceph-helpers.sh:872: get_num_active_clean: xmlstarlet sel -t -m '//pg_stat/state[(contains(.,'\''active'\'') and contains(.,'\''clean'\'') and not(contains(.,'\''stale'\'')))]' -v . -n .../qa/workunits/ceph-helpers.sh:873: get_num_active_clean: grep -v '^$' .../qa/workunits/ceph-helpers.sh:873: get_num_active_clean: wc -l ../qa/workunits/ceph-helpers.sh:993: wait_for_clean: cur_active_clean=0 ../qa/workunits/ceph-helpers.sh:994: wait_for_clean: test 0 '!=' 0 ../qa/workunits/ceph-helpers.sh:998: wait_for_clean: sleep 1 .../qa/workunits/ceph-helpers.sh:999: wait_for_clean: expr 67 + 1 ../qa/workunits/ceph-helpers.sh:999: wait_for_clean: timer=68 ../qa/workunits/ceph-helpers.sh:985: wait_for_clean: is_clean .../qa/workunits/ceph-helpers.sh:952: is_clean: get_num_pgs .../qa/workunits/ceph-helpers.sh:898: get_num_pgs: ceph --format xml status .../qa/workunits/ceph-helpers.sh:899: get_num_pgs: xmlstarlet sel -t -m //pgmap/num_pgs -v . ../qa/workunits/ceph-helpers.sh:952: is_clean: num_pgs=4 ../qa/workunits/ceph-helpers.sh:953: is_clean: test 4 '!=' 0 .../qa/workunits/ceph-helpers.sh:954: is_clean: get_num_active_clean .../qa/workunits/ceph-helpers.sh:863: get_num_active_clean: local 'expression=(' .../qa/workunits/ceph-helpers.sh:864: get_num_active_clean: expression+='contains(.,'\''active'\'') and ' .../qa/workunits/ceph-helpers.sh:865: get_num_active_clean: expression+='contains(.,'\''clean'\'') and ' .../qa/workunits/ceph-helpers.sh:866: get_num_active_clean: expression+='not(contains(.,'\''stale'\''))' .../qa/workunits/ceph-helpers.sh:867: get_num_active_clean: expression+=')' .../qa/workunits/ceph-helpers.sh:871: get_num_active_clean: ceph --format xml pg dump pgs .../qa/workunits/ceph-helpers.sh:872: get_num_active_clean: xmlstarlet sel -t -m '//pg_stat/state[(contains(.,'\''active'\'') and contains(.,'\''clean'\'') and not(contains(.,'\''stale'\'')))]' -v . -n .../qa/workunits/ceph-helpers.sh:873: get_num_active_clean: grep -v '^$' .../qa/workunits/ceph-helpers.sh:873: get_num_active_clean: wc -l ../qa/workunits/ceph-helpers.sh:954: is_clean: test 0 = 4 ../qa/workunits/ceph-helpers.sh:954: is_clean: return 1 ../qa/workunits/ceph-helpers.sh:986: wait_for_clean: get_is_making_recovery_progress .../qa/workunits/ceph-helpers.sh:839: get_is_making_recovery_progress: ceph --format xml status .../qa/workunits/ceph-helpers.sh:839: get_is_making_recovery_progress: xmlstarlet sel -t -m //pgmap/recovering_keys_per_sec -v . -o ' ' -t -m //pgmap/recovering_bytes_per_sec -v . -o ' ' -t -m //pgmap/recovering_objects_per_sec -v . ../qa/workunits/ceph-helpers.sh:839: get_is_making_recovery_progress: local progress= ../qa/workunits/ceph-helpers.sh:840: get_is_making_recovery_progress: test -n '' ../qa/workunits/ceph-helpers.sh:988: wait_for_clean: (( timer >= 120 )) .../qa/workunits/ceph-helpers.sh:993: wait_for_clean: get_num_active_clean .../qa/workunits/ceph-helpers.sh:863: get_num_active_clean: local 'expression=(' .../qa/workunits/ceph-helpers.sh:864: get_num_active_clean: expression+='contains(.,'\''active'\'') and ' .../qa/workunits/ceph-helpers.sh:865: get_num_active_clean: expression+='contains(.,'\''clean'\'') and ' .../qa/workunits/ceph-helpers.sh:866: get_num_active_clean: expression+='not(contains(.,'\''stale'\''))' .../qa/workunits/ceph-helpers.sh:867: get_num_active_clean: expression+=')' .../qa/workunits/ceph-helpers.sh:871: get_num_active_clean: ceph --format xml pg dump pgs .../qa/workunits/ceph-helpers.sh:872: get_num_active_clean: xmlstarlet sel -t -m '//pg_stat/state[(contains(.,'\''active'\'') and contains(.,'\''clean'\'') and not(contains(.,'\''stale'\'')))]' -v . -n .../qa/workunits/ceph-helpers.sh:873: get_num_active_clean: grep -v '^$' .../qa/workunits/ceph-helpers.sh:873: get_num_active_clean: wc -l ../qa/workunits/ceph-helpers.sh:993: wait_for_clean: cur_active_clean=0 ../qa/workunits/ceph-helpers.sh:994: wait_for_clean: test 0 '!=' 0 ../qa/workunits/ceph-helpers.sh:998: wait_for_clean: sleep 1 .../qa/workunits/ceph-helpers.sh:999: wait_for_clean: expr 68 + 1 ../qa/workunits/ceph-helpers.sh:999: wait_for_clean: timer=69 ../qa/workunits/ceph-helpers.sh:985: wait_for_clean: is_clean .../qa/workunits/ceph-helpers.sh:952: is_clean: get_num_pgs .../qa/workunits/ceph-helpers.sh:898: get_num_pgs: ceph --format xml status .../qa/workunits/ceph-helpers.sh:899: get_num_pgs: xmlstarlet sel -t -m //pgmap/num_pgs -v . ../qa/workunits/ceph-helpers.sh:952: is_clean: num_pgs=4 ../qa/workunits/ceph-helpers.sh:953: is_clean: test 4 '!=' 0 .../qa/workunits/ceph-helpers.sh:954: is_clean: get_num_active_clean .../qa/workunits/ceph-helpers.sh:863: get_num_active_clean: local 'expression=(' .../qa/workunits/ceph-helpers.sh:864: get_num_active_clean: expression+='contains(.,'\''active'\'') and ' .../qa/workunits/ceph-helpers.sh:865: get_num_active_clean: expression+='contains(.,'\''clean'\'') and ' .../qa/workunits/ceph-helpers.sh:866: get_num_active_clean: expression+='not(contains(.,'\''stale'\''))' .../qa/workunits/ceph-helpers.sh:867: get_num_active_clean: expression+=')' .../qa/workunits/ceph-helpers.sh:871: get_num_active_clean: ceph --format xml pg dump pgs .../qa/workunits/ceph-helpers.sh:872: get_num_active_clean: xmlstarlet sel -t -m '//pg_stat/state[(contains(.,'\''active'\'') and contains(.,'\''clean'\'') and not(contains(.,'\''stale'\'')))]' -v . -n .../qa/workunits/ceph-helpers.sh:873: get_num_active_clean: grep -v '^$' .../qa/workunits/ceph-helpers.sh:873: get_num_active_clean: wc -l ../qa/workunits/ceph-helpers.sh:954: is_clean: test 0 = 4 ../qa/workunits/ceph-helpers.sh:954: is_clean: return 1 ../qa/workunits/ceph-helpers.sh:986: wait_for_clean: get_is_making_recovery_progress .../qa/workunits/ceph-helpers.sh:839: get_is_making_recovery_progress: ceph --format xml status .../qa/workunits/ceph-helpers.sh:839: get_is_making_recovery_progress: xmlstarlet sel -t -m //pgmap/recovering_keys_per_sec -v . -o ' ' -t -m //pgmap/recovering_bytes_per_sec -v . -o ' ' -t -m //pgmap/recovering_objects_per_sec -v . ../qa/workunits/ceph-helpers.sh:839: get_is_making_recovery_progress: local progress= ../qa/workunits/ceph-helpers.sh:840: get_is_making_recovery_progress: test -n '' ../qa/workunits/ceph-helpers.sh:988: wait_for_clean: (( timer >= 120 )) .../qa/workunits/ceph-helpers.sh:993: wait_for_clean: get_num_active_clean .../qa/workunits/ceph-helpers.sh:863: get_num_active_clean: local 'expression=(' .../qa/workunits/ceph-helpers.sh:864: get_num_active_clean: expression+='contains(.,'\''active'\'') and ' .../qa/workunits/ceph-helpers.sh:865: get_num_active_clean: expression+='contains(.,'\''clean'\'') and ' .../qa/workunits/ceph-helpers.sh:866: get_num_active_clean: expression+='not(contains(.,'\''stale'\''))' .../qa/workunits/ceph-helpers.sh:867: get_num_active_clean: expression+=')' .../qa/workunits/ceph-helpers.sh:871: get_num_active_clean: ceph --format xml pg dump pgs .../qa/workunits/ceph-helpers.sh:872: get_num_active_clean: xmlstarlet sel -t -m '//pg_stat/state[(contains(.,'\''active'\'') and contains(.,'\''clean'\'') and not(contains(.,'\''stale'\'')))]' -v . -n .../qa/workunits/ceph-helpers.sh:873: get_num_active_clean: grep -v '^$' .../qa/workunits/ceph-helpers.sh:873: get_num_active_clean: wc -l ../qa/workunits/ceph-helpers.sh:993: wait_for_clean: cur_active_clean=0 ../qa/workunits/ceph-helpers.sh:994: wait_for_clean: test 0 '!=' 0 ../qa/workunits/ceph-helpers.sh:998: wait_for_clean: sleep 1 .../qa/workunits/ceph-helpers.sh:999: wait_for_clean: expr 69 + 1 ../qa/workunits/ceph-helpers.sh:999: wait_for_clean: timer=70 ../qa/workunits/ceph-helpers.sh:985: wait_for_clean: is_clean .../qa/workunits/ceph-helpers.sh:952: is_clean: get_num_pgs .../qa/workunits/ceph-helpers.sh:898: get_num_pgs: ceph --format xml status .../qa/workunits/ceph-helpers.sh:899: get_num_pgs: xmlstarlet sel -t -m //pgmap/num_pgs -v . ../qa/workunits/ceph-helpers.sh:952: is_clean: num_pgs=4 ../qa/workunits/ceph-helpers.sh:953: is_clean: test 4 '!=' 0 .../qa/workunits/ceph-helpers.sh:954: is_clean: get_num_active_clean .../qa/workunits/ceph-helpers.sh:863: get_num_active_clean: local 'expression=(' .../qa/workunits/ceph-helpers.sh:864: get_num_active_clean: expression+='contains(.,'\''active'\'') and ' .../qa/workunits/ceph-helpers.sh:865: get_num_active_clean: expression+='contains(.,'\''clean'\'') and ' .../qa/workunits/ceph-helpers.sh:866: get_num_active_clean: expression+='not(contains(.,'\''stale'\''))' .../qa/workunits/ceph-helpers.sh:867: get_num_active_clean: expression+=')' .../qa/workunits/ceph-helpers.sh:871: get_num_active_clean: ceph --format xml pg dump pgs .../qa/workunits/ceph-helpers.sh:872: get_num_active_clean: xmlstarlet sel -t -m '//pg_stat/state[(contains(.,'\''active'\'') and contains(.,'\''clean'\'') and not(contains(.,'\''stale'\'')))]' -v . -n .../qa/workunits/ceph-helpers.sh:873: get_num_active_clean: grep -v '^$' .../qa/workunits/ceph-helpers.sh:873: get_num_active_clean: wc -l ../qa/workunits/ceph-helpers.sh:954: is_clean: test 0 = 4 ../qa/workunits/ceph-helpers.sh:954: is_clean: return 1 ../qa/workunits/ceph-helpers.sh:986: wait_for_clean: get_is_making_recovery_progress .../qa/workunits/ceph-helpers.sh:839: get_is_making_recovery_progress: ceph --format xml status .../qa/workunits/ceph-helpers.sh:839: get_is_making_recovery_progress: xmlstarlet sel -t -m //pgmap/recovering_keys_per_sec -v . -o ' ' -t -m //pgmap/recovering_bytes_per_sec -v . -o ' ' -t -m //pgmap/recovering_objects_per_sec -v . ../qa/workunits/ceph-helpers.sh:839: get_is_making_recovery_progress: local progress= ../qa/workunits/ceph-helpers.sh:840: get_is_making_recovery_progress: test -n '' ../qa/workunits/ceph-helpers.sh:988: wait_for_clean: (( timer >= 120 )) .../qa/workunits/ceph-helpers.sh:993: wait_for_clean: get_num_active_clean .../qa/workunits/ceph-helpers.sh:863: get_num_active_clean: local 'expression=(' .../qa/workunits/ceph-helpers.sh:864: get_num_active_clean: expression+='contains(.,'\''active'\'') and ' .../qa/workunits/ceph-helpers.sh:865: get_num_active_clean: expression+='contains(.,'\''clean'\'') and ' .../qa/workunits/ceph-helpers.sh:866: get_num_active_clean: expression+='not(contains(.,'\''stale'\''))' .../qa/workunits/ceph-helpers.sh:867: get_num_active_clean: expression+=')' .../qa/workunits/ceph-helpers.sh:871: get_num_active_clean: ceph --format xml pg dump pgs .../qa/workunits/ceph-helpers.sh:872: get_num_active_clean: xmlstarlet sel -t -m '//pg_stat/state[(contains(.,'\''active'\'') and contains(.,'\''clean'\'') and not(contains(.,'\''stale'\'')))]' -v . -n .../qa/workunits/ceph-helpers.sh:873: get_num_active_clean: grep -v '^$' .../qa/workunits/ceph-helpers.sh:873: get_num_active_clean: wc -l ../qa/workunits/ceph-helpers.sh:993: wait_for_clean: cur_active_clean=0 ../qa/workunits/ceph-helpers.sh:994: wait_for_clean: test 0 '!=' 0 ../qa/workunits/ceph-helpers.sh:998: wait_for_clean: sleep 1 .../qa/workunits/ceph-helpers.sh:999: wait_for_clean: expr 70 + 1 ../qa/workunits/ceph-helpers.sh:999: wait_for_clean: timer=71 ../qa/workunits/ceph-helpers.sh:985: wait_for_clean: is_clean .../qa/workunits/ceph-helpers.sh:952: is_clean: get_num_pgs .../qa/workunits/ceph-helpers.sh:898: get_num_pgs: ceph --format xml status .../qa/workunits/ceph-helpers.sh:899: get_num_pgs: xmlstarlet sel -t -m //pgmap/num_pgs -v . ../qa/workunits/ceph-helpers.sh:952: is_clean: num_pgs=4 ../qa/workunits/ceph-helpers.sh:953: is_clean: test 4 '!=' 0 .../qa/workunits/ceph-helpers.sh:954: is_clean: get_num_active_clean .../qa/workunits/ceph-helpers.sh:863: get_num_active_clean: local 'expression=(' .../qa/workunits/ceph-helpers.sh:864: get_num_active_clean: expression+='contains(.,'\''active'\'') and ' .../qa/workunits/ceph-helpers.sh:865: get_num_active_clean: expression+='contains(.,'\''clean'\'') and ' .../qa/workunits/ceph-helpers.sh:866: get_num_active_clean: expression+='not(contains(.,'\''stale'\''))' .../qa/workunits/ceph-helpers.sh:867: get_num_active_clean: expression+=')' .../qa/workunits/ceph-helpers.sh:871: get_num_active_clean: ceph --format xml pg dump pgs .../qa/workunits/ceph-helpers.sh:872: get_num_active_clean: xmlstarlet sel -t -m '//pg_stat/state[(contains(.,'\''active'\'') and contains(.,'\''clean'\'') and not(contains(.,'\''stale'\'')))]' -v . -n .../qa/workunits/ceph-helpers.sh:873: get_num_active_clean: grep -v '^$' .../qa/workunits/ceph-helpers.sh:873: get_num_active_clean: wc -l ../qa/workunits/ceph-helpers.sh:954: is_clean: test 0 = 4 ../qa/workunits/ceph-helpers.sh:954: is_clean: return 1 ../qa/workunits/ceph-helpers.sh:986: wait_for_clean: get_is_making_recovery_progress .../qa/workunits/ceph-helpers.sh:839: get_is_making_recovery_progress: ceph --format xml status .../qa/workunits/ceph-helpers.sh:839: get_is_making_recovery_progress: xmlstarlet sel -t -m //pgmap/recovering_keys_per_sec -v . -o ' ' -t -m //pgmap/recovering_bytes_per_sec -v . -o ' ' -t -m //pgmap/recovering_objects_per_sec -v . ../qa/workunits/ceph-helpers.sh:839: get_is_making_recovery_progress: local progress= ../qa/workunits/ceph-helpers.sh:840: get_is_making_recovery_progress: test -n '' ../qa/workunits/ceph-helpers.sh:988: wait_for_clean: (( timer >= 120 )) .../qa/workunits/ceph-helpers.sh:993: wait_for_clean: get_num_active_clean .../qa/workunits/ceph-helpers.sh:863: get_num_active_clean: local 'expression=(' .../qa/workunits/ceph-helpers.sh:864: get_num_active_clean: expression+='contains(.,'\''active'\'') and ' .../qa/workunits/ceph-helpers.sh:865: get_num_active_clean: expression+='contains(.,'\''clean'\'') and ' .../qa/workunits/ceph-helpers.sh:866: get_num_active_clean: expression+='not(contains(.,'\''stale'\''))' .../qa/workunits/ceph-helpers.sh:867: get_num_active_clean: expression+=')' .../qa/workunits/ceph-helpers.sh:871: get_num_active_clean: ceph --format xml pg dump pgs .../qa/workunits/ceph-helpers.sh:872: get_num_active_clean: xmlstarlet sel -t -m '//pg_stat/state[(contains(.,'\''active'\'') and contains(.,'\''clean'\'') and not(contains(.,'\''stale'\'')))]' -v . -n .../qa/workunits/ceph-helpers.sh:873: get_num_active_clean: grep -v '^$' .../qa/workunits/ceph-helpers.sh:873: get_num_active_clean: wc -l ../qa/workunits/ceph-helpers.sh:993: wait_for_clean: cur_active_clean=0 ../qa/workunits/ceph-helpers.sh:994: wait_for_clean: test 0 '!=' 0 ../qa/workunits/ceph-helpers.sh:998: wait_for_clean: sleep 1 .../qa/workunits/ceph-helpers.sh:999: wait_for_clean: expr 71 + 1 ../qa/workunits/ceph-helpers.sh:999: wait_for_clean: timer=72 ../qa/workunits/ceph-helpers.sh:985: wait_for_clean: is_clean .../qa/workunits/ceph-helpers.sh:952: is_clean: get_num_pgs .../qa/workunits/ceph-helpers.sh:898: get_num_pgs: ceph --format xml status .../qa/workunits/ceph-helpers.sh:899: get_num_pgs: xmlstarlet sel -t -m //pgmap/num_pgs -v . ../qa/workunits/ceph-helpers.sh:952: is_clean: num_pgs=4 ../qa/workunits/ceph-helpers.sh:953: is_clean: test 4 '!=' 0 .../qa/workunits/ceph-helpers.sh:954: is_clean: get_num_active_clean .../qa/workunits/ceph-helpers.sh:863: get_num_active_clean: local 'expression=(' .../qa/workunits/ceph-helpers.sh:864: get_num_active_clean: expression+='contains(.,'\''active'\'') and ' .../qa/workunits/ceph-helpers.sh:865: get_num_active_clean: expression+='contains(.,'\''clean'\'') and ' .../qa/workunits/ceph-helpers.sh:866: get_num_active_clean: expression+='not(contains(.,'\''stale'\''))' .../qa/workunits/ceph-helpers.sh:867: get_num_active_clean: expression+=')' .../qa/workunits/ceph-helpers.sh:871: get_num_active_clean: ceph --format xml pg dump pgs .../qa/workunits/ceph-helpers.sh:872: get_num_active_clean: xmlstarlet sel -t -m '//pg_stat/state[(contains(.,'\''active'\'') and contains(.,'\''clean'\'') and not(contains(.,'\''stale'\'')))]' -v . -n .../qa/workunits/ceph-helpers.sh:873: get_num_active_clean: grep -v '^$' .../qa/workunits/ceph-helpers.sh:873: get_num_active_clean: wc -l ../qa/workunits/ceph-helpers.sh:954: is_clean: test 0 = 4 ../qa/workunits/ceph-helpers.sh:954: is_clean: return 1 ../qa/workunits/ceph-helpers.sh:986: wait_for_clean: get_is_making_recovery_progress .../qa/workunits/ceph-helpers.sh:839: get_is_making_recovery_progress: ceph --format xml status .../qa/workunits/ceph-helpers.sh:839: get_is_making_recovery_progress: xmlstarlet sel -t -m //pgmap/recovering_keys_per_sec -v . -o ' ' -t -m //pgmap/recovering_bytes_per_sec -v . -o ' ' -t -m //pgmap/recovering_objects_per_sec -v . ../qa/workunits/ceph-helpers.sh:839: get_is_making_recovery_progress: local progress= ../qa/workunits/ceph-helpers.sh:840: get_is_making_recovery_progress: test -n '' ../qa/workunits/ceph-helpers.sh:988: wait_for_clean: (( timer >= 120 )) .../qa/workunits/ceph-helpers.sh:993: wait_for_clean: get_num_active_clean .../qa/workunits/ceph-helpers.sh:863: get_num_active_clean: local 'expression=(' .../qa/workunits/ceph-helpers.sh:864: get_num_active_clean: expression+='contains(.,'\''active'\'') and ' .../qa/workunits/ceph-helpers.sh:865: get_num_active_clean: expression+='contains(.,'\''clean'\'') and ' .../qa/workunits/ceph-helpers.sh:866: get_num_active_clean: expression+='not(contains(.,'\''stale'\''))' .../qa/workunits/ceph-helpers.sh:867: get_num_active_clean: expression+=')' .../qa/workunits/ceph-helpers.sh:871: get_num_active_clean: ceph --format xml pg dump pgs .../qa/workunits/ceph-helpers.sh:873: get_num_active_clean: grep -v '^$' .../qa/workunits/ceph-helpers.sh:872: get_num_active_clean: xmlstarlet sel -t -m '//pg_stat/state[(contains(.,'\''active'\'') and contains(.,'\''clean'\'') and not(contains(.,'\''stale'\'')))]' -v . -n .../qa/workunits/ceph-helpers.sh:873: get_num_active_clean: wc -l ../qa/workunits/ceph-helpers.sh:993: wait_for_clean: cur_active_clean=0 ../qa/workunits/ceph-helpers.sh:994: wait_for_clean: test 0 '!=' 0 ../qa/workunits/ceph-helpers.sh:998: wait_for_clean: sleep 1 .../qa/workunits/ceph-helpers.sh:999: wait_for_clean: expr 72 + 1 ../qa/workunits/ceph-helpers.sh:999: wait_for_clean: timer=73 ../qa/workunits/ceph-helpers.sh:985: wait_for_clean: is_clean .../qa/workunits/ceph-helpers.sh:952: is_clean: get_num_pgs .../qa/workunits/ceph-helpers.sh:898: get_num_pgs: ceph --format xml status .../qa/workunits/ceph-helpers.sh:899: get_num_pgs: xmlstarlet sel -t -m //pgmap/num_pgs -v . ../qa/workunits/ceph-helpers.sh:952: is_clean: num_pgs=4 ../qa/workunits/ceph-helpers.sh:953: is_clean: test 4 '!=' 0 .../qa/workunits/ceph-helpers.sh:954: is_clean: get_num_active_clean .../qa/workunits/ceph-helpers.sh:863: get_num_active_clean: local 'expression=(' .../qa/workunits/ceph-helpers.sh:864: get_num_active_clean: expression+='contains(.,'\''active'\'') and ' .../qa/workunits/ceph-helpers.sh:865: get_num_active_clean: expression+='contains(.,'\''clean'\'') and ' .../qa/workunits/ceph-helpers.sh:866: get_num_active_clean: expression+='not(contains(.,'\''stale'\''))' .../qa/workunits/ceph-helpers.sh:867: get_num_active_clean: expression+=')' .../qa/workunits/ceph-helpers.sh:871: get_num_active_clean: ceph --format xml pg dump pgs .../qa/workunits/ceph-helpers.sh:872: get_num_active_clean: xmlstarlet sel -t -m '//pg_stat/state[(contains(.,'\''active'\'') and contains(.,'\''clean'\'') and not(contains(.,'\''stale'\'')))]' -v . -n .../qa/workunits/ceph-helpers.sh:873: get_num_active_clean: wc -l .../qa/workunits/ceph-helpers.sh:873: get_num_active_clean: grep -v '^$' ../qa/workunits/ceph-helpers.sh:954: is_clean: test 0 = 4 ../qa/workunits/ceph-helpers.sh:954: is_clean: return 1 ../qa/workunits/ceph-helpers.sh:986: wait_for_clean: get_is_making_recovery_progress .../qa/workunits/ceph-helpers.sh:839: get_is_making_recovery_progress: ceph --format xml status .../qa/workunits/ceph-helpers.sh:839: get_is_making_recovery_progress: xmlstarlet sel -t -m //pgmap/recovering_keys_per_sec -v . -o ' ' -t -m //pgmap/recovering_bytes_per_sec -v . -o ' ' -t -m //pgmap/recovering_objects_per_sec -v . ../qa/workunits/ceph-helpers.sh:839: get_is_making_recovery_progress: local progress= ../qa/workunits/ceph-helpers.sh:840: get_is_making_recovery_progress: test -n '' ../qa/workunits/ceph-helpers.sh:988: wait_for_clean: (( timer >= 120 )) .../qa/workunits/ceph-helpers.sh:993: wait_for_clean: get_num_active_clean .../qa/workunits/ceph-helpers.sh:863: get_num_active_clean: local 'expression=(' .../qa/workunits/ceph-helpers.sh:864: get_num_active_clean: expression+='contains(.,'\''active'\'') and ' .../qa/workunits/ceph-helpers.sh:865: get_num_active_clean: expression+='contains(.,'\''clean'\'') and ' .../qa/workunits/ceph-helpers.sh:866: get_num_active_clean: expression+='not(contains(.,'\''stale'\''))' .../qa/workunits/ceph-helpers.sh:867: get_num_active_clean: expression+=')' .../qa/workunits/ceph-helpers.sh:871: get_num_active_clean: ceph --format xml pg dump pgs .../qa/workunits/ceph-helpers.sh:872: get_num_active_clean: xmlstarlet sel -t -m '//pg_stat/state[(contains(.,'\''active'\'') and contains(.,'\''clean'\'') and not(contains(.,'\''stale'\'')))]' -v . -n .../qa/workunits/ceph-helpers.sh:873: get_num_active_clean: grep -v '^$' .../qa/workunits/ceph-helpers.sh:873: get_num_active_clean: wc -l ../qa/workunits/ceph-helpers.sh:993: wait_for_clean: cur_active_clean=0 ../qa/workunits/ceph-helpers.sh:994: wait_for_clean: test 0 '!=' 0 ../qa/workunits/ceph-helpers.sh:998: wait_for_clean: sleep 1 .../qa/workunits/ceph-helpers.sh:999: wait_for_clean: expr 73 + 1 ../qa/workunits/ceph-helpers.sh:999: wait_for_clean: timer=74 ../qa/workunits/ceph-helpers.sh:985: wait_for_clean: is_clean .../qa/workunits/ceph-helpers.sh:952: is_clean: get_num_pgs .../qa/workunits/ceph-helpers.sh:898: get_num_pgs: ceph --format xml status .../qa/workunits/ceph-helpers.sh:899: get_num_pgs: xmlstarlet sel -t -m //pgmap/num_pgs -v . ../qa/workunits/ceph-helpers.sh:952: is_clean: num_pgs=4 ../qa/workunits/ceph-helpers.sh:953: is_clean: test 4 '!=' 0 .../qa/workunits/ceph-helpers.sh:954: is_clean: get_num_active_clean .../qa/workunits/ceph-helpers.sh:863: get_num_active_clean: local 'expression=(' .../qa/workunits/ceph-helpers.sh:864: get_num_active_clean: expression+='contains(.,'\''active'\'') and ' .../qa/workunits/ceph-helpers.sh:865: get_num_active_clean: expression+='contains(.,'\''clean'\'') and ' .../qa/workunits/ceph-helpers.sh:866: get_num_active_clean: expression+='not(contains(.,'\''stale'\''))' .../qa/workunits/ceph-helpers.sh:867: get_num_active_clean: expression+=')' .../qa/workunits/ceph-helpers.sh:871: get_num_active_clean: ceph --format xml pg dump pgs .../qa/workunits/ceph-helpers.sh:872: get_num_active_clean: xmlstarlet sel -t -m '//pg_stat/state[(contains(.,'\''active'\'') and contains(.,'\''clean'\'') and not(contains(.,'\''stale'\'')))]' -v . -n .../qa/workunits/ceph-helpers.sh:873: get_num_active_clean: grep -v '^$' .../qa/workunits/ceph-helpers.sh:873: get_num_active_clean: wc -l ../qa/workunits/ceph-helpers.sh:954: is_clean: test 0 = 4 ../qa/workunits/ceph-helpers.sh:954: is_clean: return 1 ../qa/workunits/ceph-helpers.sh:986: wait_for_clean: get_is_making_recovery_progress .../qa/workunits/ceph-helpers.sh:839: get_is_making_recovery_progress: ceph --format xml status .../qa/workunits/ceph-helpers.sh:839: get_is_making_recovery_progress: xmlstarlet sel -t -m //pgmap/recovering_keys_per_sec -v . -o ' ' -t -m //pgmap/recovering_bytes_per_sec -v . -o ' ' -t -m //pgmap/recovering_objects_per_sec -v . ../qa/workunits/ceph-helpers.sh:839: get_is_making_recovery_progress: local progress= ../qa/workunits/ceph-helpers.sh:840: get_is_making_recovery_progress: test -n '' ../qa/workunits/ceph-helpers.sh:988: wait_for_clean: (( timer >= 120 )) .../qa/workunits/ceph-helpers.sh:993: wait_for_clean: get_num_active_clean .../qa/workunits/ceph-helpers.sh:863: get_num_active_clean: local 'expression=(' .../qa/workunits/ceph-helpers.sh:864: get_num_active_clean: expression+='contains(.,'\''active'\'') and ' .../qa/workunits/ceph-helpers.sh:865: get_num_active_clean: expression+='contains(.,'\''clean'\'') and ' .../qa/workunits/ceph-helpers.sh:866: get_num_active_clean: expression+='not(contains(.,'\''stale'\''))' .../qa/workunits/ceph-helpers.sh:867: get_num_active_clean: expression+=')' .../qa/workunits/ceph-helpers.sh:871: get_num_active_clean: ceph --format xml pg dump pgs .../qa/workunits/ceph-helpers.sh:872: get_num_active_clean: xmlstarlet sel -t -m '//pg_stat/state[(contains(.,'\''active'\'') and contains(.,'\''clean'\'') and not(contains(.,'\''stale'\'')))]' -v . -n .../qa/workunits/ceph-helpers.sh:873: get_num_active_clean: grep -v '^$' .../qa/workunits/ceph-helpers.sh:873: get_num_active_clean: wc -l ../qa/workunits/ceph-helpers.sh:993: wait_for_clean: cur_active_clean=0 ../qa/workunits/ceph-helpers.sh:994: wait_for_clean: test 0 '!=' 0 ../qa/workunits/ceph-helpers.sh:998: wait_for_clean: sleep 1 .../qa/workunits/ceph-helpers.sh:999: wait_for_clean: expr 74 + 1 ../qa/workunits/ceph-helpers.sh:999: wait_for_clean: timer=75 ../qa/workunits/ceph-helpers.sh:985: wait_for_clean: is_clean .../qa/workunits/ceph-helpers.sh:952: is_clean: get_num_pgs .../qa/workunits/ceph-helpers.sh:898: get_num_pgs: ceph --format xml status .../qa/workunits/ceph-helpers.sh:899: get_num_pgs: xmlstarlet sel -t -m //pgmap/num_pgs -v . ../qa/workunits/ceph-helpers.sh:952: is_clean: num_pgs=4 ../qa/workunits/ceph-helpers.sh:953: is_clean: test 4 '!=' 0 .../qa/workunits/ceph-helpers.sh:954: is_clean: get_num_active_clean .../qa/workunits/ceph-helpers.sh:863: get_num_active_clean: local 'expression=(' .../qa/workunits/ceph-helpers.sh:864: get_num_active_clean: expression+='contains(.,'\''active'\'') and ' .../qa/workunits/ceph-helpers.sh:865: get_num_active_clean: expression+='contains(.,'\''clean'\'') and ' .../qa/workunits/ceph-helpers.sh:866: get_num_active_clean: expression+='not(contains(.,'\''stale'\''))' .../qa/workunits/ceph-helpers.sh:867: get_num_active_clean: expression+=')' .../qa/workunits/ceph-helpers.sh:871: get_num_active_clean: ceph --format xml pg dump pgs .../qa/workunits/ceph-helpers.sh:872: get_num_active_clean: xmlstarlet sel -t -m '//pg_stat/state[(contains(.,'\''active'\'') and contains(.,'\''clean'\'') and not(contains(.,'\''stale'\'')))]' -v . -n .../qa/workunits/ceph-helpers.sh:873: get_num_active_clean: wc -l .../qa/workunits/ceph-helpers.sh:873: get_num_active_clean: grep -v '^$' ../qa/workunits/ceph-helpers.sh:954: is_clean: test 0 = 4 ../qa/workunits/ceph-helpers.sh:954: is_clean: return 1 ../qa/workunits/ceph-helpers.sh:986: wait_for_clean: get_is_making_recovery_progress .../qa/workunits/ceph-helpers.sh:839: get_is_making_recovery_progress: ceph --format xml status .../qa/workunits/ceph-helpers.sh:839: get_is_making_recovery_progress: xmlstarlet sel -t -m //pgmap/recovering_keys_per_sec -v . -o ' ' -t -m //pgmap/recovering_bytes_per_sec -v . -o ' ' -t -m //pgmap/recovering_objects_per_sec -v . ../qa/workunits/ceph-helpers.sh:839: get_is_making_recovery_progress: local progress= ../qa/workunits/ceph-helpers.sh:840: get_is_making_recovery_progress: test -n '' ../qa/workunits/ceph-helpers.sh:988: wait_for_clean: (( timer >= 120 )) .../qa/workunits/ceph-helpers.sh:993: wait_for_clean: get_num_active_clean .../qa/workunits/ceph-helpers.sh:863: get_num_active_clean: local 'expression=(' .../qa/workunits/ceph-helpers.sh:864: get_num_active_clean: expression+='contains(.,'\''active'\'') and ' .../qa/workunits/ceph-helpers.sh:865: get_num_active_clean: expression+='contains(.,'\''clean'\'') and ' .../qa/workunits/ceph-helpers.sh:866: get_num_active_clean: expression+='not(contains(.,'\''stale'\''))' .../qa/workunits/ceph-helpers.sh:867: get_num_active_clean: expression+=')' .../qa/workunits/ceph-helpers.sh:871: get_num_active_clean: ceph --format xml pg dump pgs .../qa/workunits/ceph-helpers.sh:872: get_num_active_clean: xmlstarlet sel -t -m '//pg_stat/state[(contains(.,'\''active'\'') and contains(.,'\''clean'\'') and not(contains(.,'\''stale'\'')))]' -v . -n .../qa/workunits/ceph-helpers.sh:873: get_num_active_clean: wc -l .../qa/workunits/ceph-helpers.sh:873: get_num_active_clean: grep -v '^$' ../qa/workunits/ceph-helpers.sh:993: wait_for_clean: cur_active_clean=0 ../qa/workunits/ceph-helpers.sh:994: wait_for_clean: test 0 '!=' 0 ../qa/workunits/ceph-helpers.sh:998: wait_for_clean: sleep 1 .../qa/workunits/ceph-helpers.sh:999: wait_for_clean: expr 75 + 1 ../qa/workunits/ceph-helpers.sh:999: wait_for_clean: timer=76 ../qa/workunits/ceph-helpers.sh:985: wait_for_clean: is_clean .../qa/workunits/ceph-helpers.sh:952: is_clean: get_num_pgs .../qa/workunits/ceph-helpers.sh:898: get_num_pgs: ceph --format xml status .../qa/workunits/ceph-helpers.sh:899: get_num_pgs: xmlstarlet sel -t -m //pgmap/num_pgs -v . ../qa/workunits/ceph-helpers.sh:952: is_clean: num_pgs=4 ../qa/workunits/ceph-helpers.sh:953: is_clean: test 4 '!=' 0 .../qa/workunits/ceph-helpers.sh:954: is_clean: get_num_active_clean .../qa/workunits/ceph-helpers.sh:863: get_num_active_clean: local 'expression=(' .../qa/workunits/ceph-helpers.sh:864: get_num_active_clean: expression+='contains(.,'\''active'\'') and ' .../qa/workunits/ceph-helpers.sh:865: get_num_active_clean: expression+='contains(.,'\''clean'\'') and ' .../qa/workunits/ceph-helpers.sh:866: get_num_active_clean: expression+='not(contains(.,'\''stale'\''))' .../qa/workunits/ceph-helpers.sh:867: get_num_active_clean: expression+=')' .../qa/workunits/ceph-helpers.sh:871: get_num_active_clean: ceph --format xml pg dump pgs .../qa/workunits/ceph-helpers.sh:872: get_num_active_clean: xmlstarlet sel -t -m '//pg_stat/state[(contains(.,'\''active'\'') and contains(.,'\''clean'\'') and not(contains(.,'\''stale'\'')))]' -v . -n .../qa/workunits/ceph-helpers.sh:873: get_num_active_clean: grep -v '^$' .../qa/workunits/ceph-helpers.sh:873: get_num_active_clean: wc -l ../qa/workunits/ceph-helpers.sh:954: is_clean: test 0 = 4 ../qa/workunits/ceph-helpers.sh:954: is_clean: return 1 ../qa/workunits/ceph-helpers.sh:986: wait_for_clean: get_is_making_recovery_progress .../qa/workunits/ceph-helpers.sh:839: get_is_making_recovery_progress: ceph --format xml status .../qa/workunits/ceph-helpers.sh:839: get_is_making_recovery_progress: xmlstarlet sel -t -m //pgmap/recovering_keys_per_sec -v . -o ' ' -t -m //pgmap/recovering_bytes_per_sec -v . -o ' ' -t -m //pgmap/recovering_objects_per_sec -v . ../qa/workunits/ceph-helpers.sh:839: get_is_making_recovery_progress: local progress= ../qa/workunits/ceph-helpers.sh:840: get_is_making_recovery_progress: test -n '' ../qa/workunits/ceph-helpers.sh:988: wait_for_clean: (( timer >= 120 )) .../qa/workunits/ceph-helpers.sh:993: wait_for_clean: get_num_active_clean .../qa/workunits/ceph-helpers.sh:863: get_num_active_clean: local 'expression=(' .../qa/workunits/ceph-helpers.sh:864: get_num_active_clean: expression+='contains(.,'\''active'\'') and ' .../qa/workunits/ceph-helpers.sh:865: get_num_active_clean: expression+='contains(.,'\''clean'\'') and ' .../qa/workunits/ceph-helpers.sh:866: get_num_active_clean: expression+='not(contains(.,'\''stale'\''))' .../qa/workunits/ceph-helpers.sh:867: get_num_active_clean: expression+=')' .../qa/workunits/ceph-helpers.sh:871: get_num_active_clean: ceph --format xml pg dump pgs .../qa/workunits/ceph-helpers.sh:872: get_num_active_clean: xmlstarlet sel -t -m '//pg_stat/state[(contains(.,'\''active'\'') and contains(.,'\''clean'\'') and not(contains(.,'\''stale'\'')))]' -v . -n .../qa/workunits/ceph-helpers.sh:873: get_num_active_clean: grep -v '^$' .../qa/workunits/ceph-helpers.sh:873: get_num_active_clean: wc -l ../qa/workunits/ceph-helpers.sh:993: wait_for_clean: cur_active_clean=0 ../qa/workunits/ceph-helpers.sh:994: wait_for_clean: test 0 '!=' 0 ../qa/workunits/ceph-helpers.sh:998: wait_for_clean: sleep 1 .../qa/workunits/ceph-helpers.sh:999: wait_for_clean: expr 76 + 1 ../qa/workunits/ceph-helpers.sh:999: wait_for_clean: timer=77 ../qa/workunits/ceph-helpers.sh:985: wait_for_clean: is_clean .../qa/workunits/ceph-helpers.sh:952: is_clean: get_num_pgs .../qa/workunits/ceph-helpers.sh:898: get_num_pgs: ceph --format xml status .../qa/workunits/ceph-helpers.sh:899: get_num_pgs: xmlstarlet sel -t -m //pgmap/num_pgs -v . ../qa/workunits/ceph-helpers.sh:952: is_clean: num_pgs=4 ../qa/workunits/ceph-helpers.sh:953: is_clean: test 4 '!=' 0 .../qa/workunits/ceph-helpers.sh:954: is_clean: get_num_active_clean .../qa/workunits/ceph-helpers.sh:863: get_num_active_clean: local 'expression=(' .../qa/workunits/ceph-helpers.sh:864: get_num_active_clean: expression+='contains(.,'\''active'\'') and ' .../qa/workunits/ceph-helpers.sh:865: get_num_active_clean: expression+='contains(.,'\''clean'\'') and ' .../qa/workunits/ceph-helpers.sh:866: get_num_active_clean: expression+='not(contains(.,'\''stale'\''))' .../qa/workunits/ceph-helpers.sh:867: get_num_active_clean: expression+=')' .../qa/workunits/ceph-helpers.sh:871: get_num_active_clean: ceph --format xml pg dump pgs .../qa/workunits/ceph-helpers.sh:872: get_num_active_clean: xmlstarlet sel -t -m '//pg_stat/state[(contains(.,'\''active'\'') and contains(.,'\''clean'\'') and not(contains(.,'\''stale'\'')))]' -v . -n .../qa/workunits/ceph-helpers.sh:873: get_num_active_clean: grep -v '^$' .../qa/workunits/ceph-helpers.sh:873: get_num_active_clean: wc -l ../qa/workunits/ceph-helpers.sh:954: is_clean: test 0 = 4 ../qa/workunits/ceph-helpers.sh:954: is_clean: return 1 ../qa/workunits/ceph-helpers.sh:986: wait_for_clean: get_is_making_recovery_progress .../qa/workunits/ceph-helpers.sh:839: get_is_making_recovery_progress: ceph --format xml status .../qa/workunits/ceph-helpers.sh:839: get_is_making_recovery_progress: xmlstarlet sel -t -m //pgmap/recovering_keys_per_sec -v . -o ' ' -t -m //pgmap/recovering_bytes_per_sec -v . -o ' ' -t -m //pgmap/recovering_objects_per_sec -v . ../qa/workunits/ceph-helpers.sh:839: get_is_making_recovery_progress: local progress= ../qa/workunits/ceph-helpers.sh:840: get_is_making_recovery_progress: test -n '' ../qa/workunits/ceph-helpers.sh:988: wait_for_clean: (( timer >= 120 )) .../qa/workunits/ceph-helpers.sh:993: wait_for_clean: get_num_active_clean .../qa/workunits/ceph-helpers.sh:863: get_num_active_clean: local 'expression=(' .../qa/workunits/ceph-helpers.sh:864: get_num_active_clean: expression+='contains(.,'\''active'\'') and ' .../qa/workunits/ceph-helpers.sh:865: get_num_active_clean: expression+='contains(.,'\''clean'\'') and ' .../qa/workunits/ceph-helpers.sh:866: get_num_active_clean: expression+='not(contains(.,'\''stale'\''))' .../qa/workunits/ceph-helpers.sh:867: get_num_active_clean: expression+=')' .../qa/workunits/ceph-helpers.sh:871: get_num_active_clean: ceph --format xml pg dump pgs .../qa/workunits/ceph-helpers.sh:872: get_num_active_clean: xmlstarlet sel -t -m '//pg_stat/state[(contains(.,'\''active'\'') and contains(.,'\''clean'\'') and not(contains(.,'\''stale'\'')))]' -v . -n .../qa/workunits/ceph-helpers.sh:873: get_num_active_clean: grep -v '^$' .../qa/workunits/ceph-helpers.sh:873: get_num_active_clean: wc -l ../qa/workunits/ceph-helpers.sh:993: wait_for_clean: cur_active_clean=0 ../qa/workunits/ceph-helpers.sh:994: wait_for_clean: test 0 '!=' 0 ../qa/workunits/ceph-helpers.sh:998: wait_for_clean: sleep 1 .../qa/workunits/ceph-helpers.sh:999: wait_for_clean: expr 77 + 1 ../qa/workunits/ceph-helpers.sh:999: wait_for_clean: timer=78 ../qa/workunits/ceph-helpers.sh:985: wait_for_clean: is_clean .../qa/workunits/ceph-helpers.sh:952: is_clean: get_num_pgs .../qa/workunits/ceph-helpers.sh:898: get_num_pgs: ceph --format xml status .../qa/workunits/ceph-helpers.sh:899: get_num_pgs: xmlstarlet sel -t -m //pgmap/num_pgs -v . ../qa/workunits/ceph-helpers.sh:952: is_clean: num_pgs=4 ../qa/workunits/ceph-helpers.sh:953: is_clean: test 4 '!=' 0 .../qa/workunits/ceph-helpers.sh:954: is_clean: get_num_active_clean .../qa/workunits/ceph-helpers.sh:863: get_num_active_clean: local 'expression=(' .../qa/workunits/ceph-helpers.sh:864: get_num_active_clean: expression+='contains(.,'\''active'\'') and ' .../qa/workunits/ceph-helpers.sh:865: get_num_active_clean: expression+='contains(.,'\''clean'\'') and ' .../qa/workunits/ceph-helpers.sh:866: get_num_active_clean: expression+='not(contains(.,'\''stale'\''))' .../qa/workunits/ceph-helpers.sh:867: get_num_active_clean: expression+=')' .../qa/workunits/ceph-helpers.sh:871: get_num_active_clean: ceph --format xml pg dump pgs .../qa/workunits/ceph-helpers.sh:872: get_num_active_clean: xmlstarlet sel -t -m '//pg_stat/state[(contains(.,'\''active'\'') and contains(.,'\''clean'\'') and not(contains(.,'\''stale'\'')))]' -v . -n .../qa/workunits/ceph-helpers.sh:873: get_num_active_clean: grep -v '^$' .../qa/workunits/ceph-helpers.sh:873: get_num_active_clean: wc -l ../qa/workunits/ceph-helpers.sh:954: is_clean: test 0 = 4 ../qa/workunits/ceph-helpers.sh:954: is_clean: return 1 ../qa/workunits/ceph-helpers.sh:986: wait_for_clean: get_is_making_recovery_progress .../qa/workunits/ceph-helpers.sh:839: get_is_making_recovery_progress: ceph --format xml status .../qa/workunits/ceph-helpers.sh:839: get_is_making_recovery_progress: xmlstarlet sel -t -m //pgmap/recovering_keys_per_sec -v . -o ' ' -t -m //pgmap/recovering_bytes_per_sec -v . -o ' ' -t -m //pgmap/recovering_objects_per_sec -v . ../qa/workunits/ceph-helpers.sh:839: get_is_making_recovery_progress: local progress= ../qa/workunits/ceph-helpers.sh:840: get_is_making_recovery_progress: test -n '' ../qa/workunits/ceph-helpers.sh:988: wait_for_clean: (( timer >= 120 )) .../qa/workunits/ceph-helpers.sh:993: wait_for_clean: get_num_active_clean .../qa/workunits/ceph-helpers.sh:863: get_num_active_clean: local 'expression=(' .../qa/workunits/ceph-helpers.sh:864: get_num_active_clean: expression+='contains(.,'\''active'\'') and ' .../qa/workunits/ceph-helpers.sh:865: get_num_active_clean: expression+='contains(.,'\''clean'\'') and ' .../qa/workunits/ceph-helpers.sh:866: get_num_active_clean: expression+='not(contains(.,'\''stale'\''))' .../qa/workunits/ceph-helpers.sh:867: get_num_active_clean: expression+=')' .../qa/workunits/ceph-helpers.sh:871: get_num_active_clean: ceph --format xml pg dump pgs .../qa/workunits/ceph-helpers.sh:872: get_num_active_clean: xmlstarlet sel -t -m '//pg_stat/state[(contains(.,'\''active'\'') and contains(.,'\''clean'\'') and not(contains(.,'\''stale'\'')))]' -v . -n .../qa/workunits/ceph-helpers.sh:873: get_num_active_clean: grep -v '^$' .../qa/workunits/ceph-helpers.sh:873: get_num_active_clean: wc -l ../qa/workunits/ceph-helpers.sh:993: wait_for_clean: cur_active_clean=0 ../qa/workunits/ceph-helpers.sh:994: wait_for_clean: test 0 '!=' 0 ../qa/workunits/ceph-helpers.sh:998: wait_for_clean: sleep 1 .../qa/workunits/ceph-helpers.sh:999: wait_for_clean: expr 78 + 1 ../qa/workunits/ceph-helpers.sh:999: wait_for_clean: timer=79 ../qa/workunits/ceph-helpers.sh:985: wait_for_clean: is_clean .../qa/workunits/ceph-helpers.sh:952: is_clean: get_num_pgs .../qa/workunits/ceph-helpers.sh:898: get_num_pgs: ceph --format xml status .../qa/workunits/ceph-helpers.sh:899: get_num_pgs: xmlstarlet sel -t -m //pgmap/num_pgs -v . ../qa/workunits/ceph-helpers.sh:952: is_clean: num_pgs=4 ../qa/workunits/ceph-helpers.sh:953: is_clean: test 4 '!=' 0 .../qa/workunits/ceph-helpers.sh:954: is_clean: get_num_active_clean .../qa/workunits/ceph-helpers.sh:863: get_num_active_clean: local 'expression=(' .../qa/workunits/ceph-helpers.sh:864: get_num_active_clean: expression+='contains(.,'\''active'\'') and ' .../qa/workunits/ceph-helpers.sh:865: get_num_active_clean: expression+='contains(.,'\''clean'\'') and ' .../qa/workunits/ceph-helpers.sh:866: get_num_active_clean: expression+='not(contains(.,'\''stale'\''))' .../qa/workunits/ceph-helpers.sh:867: get_num_active_clean: expression+=')' .../qa/workunits/ceph-helpers.sh:871: get_num_active_clean: ceph --format xml pg dump pgs .../qa/workunits/ceph-helpers.sh:872: get_num_active_clean: xmlstarlet sel -t -m '//pg_stat/state[(contains(.,'\''active'\'') and contains(.,'\''clean'\'') and not(contains(.,'\''stale'\'')))]' -v . -n .../qa/workunits/ceph-helpers.sh:873: get_num_active_clean: grep -v '^$' .../qa/workunits/ceph-helpers.sh:873: get_num_active_clean: wc -l ../qa/workunits/ceph-helpers.sh:954: is_clean: test 0 = 4 ../qa/workunits/ceph-helpers.sh:954: is_clean: return 1 ../qa/workunits/ceph-helpers.sh:986: wait_for_clean: get_is_making_recovery_progress .../qa/workunits/ceph-helpers.sh:839: get_is_making_recovery_progress: ceph --format xml status .../qa/workunits/ceph-helpers.sh:839: get_is_making_recovery_progress: xmlstarlet sel -t -m //pgmap/recovering_keys_per_sec -v . -o ' ' -t -m //pgmap/recovering_bytes_per_sec -v . -o ' ' -t -m //pgmap/recovering_objects_per_sec -v . ../qa/workunits/ceph-helpers.sh:839: get_is_making_recovery_progress: local progress= ../qa/workunits/ceph-helpers.sh:840: get_is_making_recovery_progress: test -n '' ../qa/workunits/ceph-helpers.sh:988: wait_for_clean: (( timer >= 120 )) .../qa/workunits/ceph-helpers.sh:993: wait_for_clean: get_num_active_clean .../qa/workunits/ceph-helpers.sh:863: get_num_active_clean: local 'expression=(' .../qa/workunits/ceph-helpers.sh:864: get_num_active_clean: expression+='contains(.,'\''active'\'') and ' .../qa/workunits/ceph-helpers.sh:865: get_num_active_clean: expression+='contains(.,'\''clean'\'') and ' .../qa/workunits/ceph-helpers.sh:866: get_num_active_clean: expression+='not(contains(.,'\''stale'\''))' .../qa/workunits/ceph-helpers.sh:867: get_num_active_clean: expression+=')' .../qa/workunits/ceph-helpers.sh:871: get_num_active_clean: ceph --format xml pg dump pgs .../qa/workunits/ceph-helpers.sh:873: get_num_active_clean: grep -v '^$' .../qa/workunits/ceph-helpers.sh:872: get_num_active_clean: xmlstarlet sel -t -m '//pg_stat/state[(contains(.,'\''active'\'') and contains(.,'\''clean'\'') and not(contains(.,'\''stale'\'')))]' -v . -n .../qa/workunits/ceph-helpers.sh:873: get_num_active_clean: wc -l ../qa/workunits/ceph-helpers.sh:993: wait_for_clean: cur_active_clean=0 ../qa/workunits/ceph-helpers.sh:994: wait_for_clean: test 0 '!=' 0 ../qa/workunits/ceph-helpers.sh:998: wait_for_clean: sleep 1 .../qa/workunits/ceph-helpers.sh:999: wait_for_clean: expr 79 + 1 ../qa/workunits/ceph-helpers.sh:999: wait_for_clean: timer=80 ../qa/workunits/ceph-helpers.sh:985: wait_for_clean: is_clean .../qa/workunits/ceph-helpers.sh:952: is_clean: get_num_pgs .../qa/workunits/ceph-helpers.sh:898: get_num_pgs: ceph --format xml status .../qa/workunits/ceph-helpers.sh:899: get_num_pgs: xmlstarlet sel -t -m //pgmap/num_pgs -v . ../qa/workunits/ceph-helpers.sh:952: is_clean: num_pgs=4 ../qa/workunits/ceph-helpers.sh:953: is_clean: test 4 '!=' 0 .../qa/workunits/ceph-helpers.sh:954: is_clean: get_num_active_clean .../qa/workunits/ceph-helpers.sh:863: get_num_active_clean: local 'expression=(' .../qa/workunits/ceph-helpers.sh:864: get_num_active_clean: expression+='contains(.,'\''active'\'') and ' .../qa/workunits/ceph-helpers.sh:865: get_num_active_clean: expression+='contains(.,'\''clean'\'') and ' .../qa/workunits/ceph-helpers.sh:866: get_num_active_clean: expression+='not(contains(.,'\''stale'\''))' .../qa/workunits/ceph-helpers.sh:867: get_num_active_clean: expression+=')' .../qa/workunits/ceph-helpers.sh:871: get_num_active_clean: ceph --format xml pg dump pgs .../qa/workunits/ceph-helpers.sh:872: get_num_active_clean: xmlstarlet sel -t -m '//pg_stat/state[(contains(.,'\''active'\'') and contains(.,'\''clean'\'') and not(contains(.,'\''stale'\'')))]' -v . -n .../qa/workunits/ceph-helpers.sh:873: get_num_active_clean: wc -l .../qa/workunits/ceph-helpers.sh:873: get_num_active_clean: grep -v '^$' ../qa/workunits/ceph-helpers.sh:954: is_clean: test 0 = 4 ../qa/workunits/ceph-helpers.sh:954: is_clean: return 1 ../qa/workunits/ceph-helpers.sh:986: wait_for_clean: get_is_making_recovery_progress .../qa/workunits/ceph-helpers.sh:839: get_is_making_recovery_progress: ceph --format xml status .../qa/workunits/ceph-helpers.sh:839: get_is_making_recovery_progress: xmlstarlet sel -t -m //pgmap/recovering_keys_per_sec -v . -o ' ' -t -m //pgmap/recovering_bytes_per_sec -v . -o ' ' -t -m //pgmap/recovering_objects_per_sec -v . ../qa/workunits/ceph-helpers.sh:839: get_is_making_recovery_progress: local progress= ../qa/workunits/ceph-helpers.sh:840: get_is_making_recovery_progress: test -n '' ../qa/workunits/ceph-helpers.sh:988: wait_for_clean: (( timer >= 120 )) .../qa/workunits/ceph-helpers.sh:993: wait_for_clean: get_num_active_clean .../qa/workunits/ceph-helpers.sh:863: get_num_active_clean: local 'expression=(' .../qa/workunits/ceph-helpers.sh:864: get_num_active_clean: expression+='contains(.,'\''active'\'') and ' .../qa/workunits/ceph-helpers.sh:865: get_num_active_clean: expression+='contains(.,'\''clean'\'') and ' .../qa/workunits/ceph-helpers.sh:866: get_num_active_clean: expression+='not(contains(.,'\''stale'\''))' .../qa/workunits/ceph-helpers.sh:867: get_num_active_clean: expression+=')' .../qa/workunits/ceph-helpers.sh:871: get_num_active_clean: ceph --format xml pg dump pgs .../qa/workunits/ceph-helpers.sh:872: get_num_active_clean: xmlstarlet sel -t -m '//pg_stat/state[(contains(.,'\''active'\'') and contains(.,'\''clean'\'') and not(contains(.,'\''stale'\'')))]' -v . -n .../qa/workunits/ceph-helpers.sh:873: get_num_active_clean: grep -v '^$' .../qa/workunits/ceph-helpers.sh:873: get_num_active_clean: wc -l ../qa/workunits/ceph-helpers.sh:993: wait_for_clean: cur_active_clean=0 ../qa/workunits/ceph-helpers.sh:994: wait_for_clean: test 0 '!=' 0 ../qa/workunits/ceph-helpers.sh:998: wait_for_clean: sleep 1 .../qa/workunits/ceph-helpers.sh:999: wait_for_clean: expr 80 + 1 ../qa/workunits/ceph-helpers.sh:999: wait_for_clean: timer=81 ../qa/workunits/ceph-helpers.sh:985: wait_for_clean: is_clean .../qa/workunits/ceph-helpers.sh:952: is_clean: get_num_pgs .../qa/workunits/ceph-helpers.sh:898: get_num_pgs: ceph --format xml status .../qa/workunits/ceph-helpers.sh:899: get_num_pgs: xmlstarlet sel -t -m //pgmap/num_pgs -v . ../qa/workunits/ceph-helpers.sh:952: is_clean: num_pgs=4 ../qa/workunits/ceph-helpers.sh:953: is_clean: test 4 '!=' 0 .../qa/workunits/ceph-helpers.sh:954: is_clean: get_num_active_clean .../qa/workunits/ceph-helpers.sh:863: get_num_active_clean: local 'expression=(' .../qa/workunits/ceph-helpers.sh:864: get_num_active_clean: expression+='contains(.,'\''active'\'') and ' .../qa/workunits/ceph-helpers.sh:865: get_num_active_clean: expression+='contains(.,'\''clean'\'') and ' .../qa/workunits/ceph-helpers.sh:866: get_num_active_clean: expression+='not(contains(.,'\''stale'\''))' .../qa/workunits/ceph-helpers.sh:867: get_num_active_clean: expression+=')' .../qa/workunits/ceph-helpers.sh:871: get_num_active_clean: ceph --format xml pg dump pgs .../qa/workunits/ceph-helpers.sh:872: get_num_active_clean: xmlstarlet sel -t -m '//pg_stat/state[(contains(.,'\''active'\'') and contains(.,'\''clean'\'') and not(contains(.,'\''stale'\'')))]' -v . -n .../qa/workunits/ceph-helpers.sh:873: get_num_active_clean: grep -v '^$' .../qa/workunits/ceph-helpers.sh:873: get_num_active_clean: wc -l ../qa/workunits/ceph-helpers.sh:954: is_clean: test 0 = 4 ../qa/workunits/ceph-helpers.sh:954: is_clean: return 1 ../qa/workunits/ceph-helpers.sh:986: wait_for_clean: get_is_making_recovery_progress .../qa/workunits/ceph-helpers.sh:839: get_is_making_recovery_progress: ceph --format xml status .../qa/workunits/ceph-helpers.sh:839: get_is_making_recovery_progress: xmlstarlet sel -t -m //pgmap/recovering_keys_per_sec -v . -o ' ' -t -m //pgmap/recovering_bytes_per_sec -v . -o ' ' -t -m //pgmap/recovering_objects_per_sec -v . ../qa/workunits/ceph-helpers.sh:839: get_is_making_recovery_progress: local progress= ../qa/workunits/ceph-helpers.sh:840: get_is_making_recovery_progress: test -n '' ../qa/workunits/ceph-helpers.sh:988: wait_for_clean: (( timer >= 120 )) .../qa/workunits/ceph-helpers.sh:993: wait_for_clean: get_num_active_clean .../qa/workunits/ceph-helpers.sh:863: get_num_active_clean: local 'expression=(' .../qa/workunits/ceph-helpers.sh:864: get_num_active_clean: expression+='contains(.,'\''active'\'') and ' .../qa/workunits/ceph-helpers.sh:865: get_num_active_clean: expression+='contains(.,'\''clean'\'') and ' .../qa/workunits/ceph-helpers.sh:866: get_num_active_clean: expression+='not(contains(.,'\''stale'\''))' .../qa/workunits/ceph-helpers.sh:867: get_num_active_clean: expression+=')' .../qa/workunits/ceph-helpers.sh:871: get_num_active_clean: ceph --format xml pg dump pgs .../qa/workunits/ceph-helpers.sh:872: get_num_active_clean: xmlstarlet sel -t -m '//pg_stat/state[(contains(.,'\''active'\'') and contains(.,'\''clean'\'') and not(contains(.,'\''stale'\'')))]' -v . -n .../qa/workunits/ceph-helpers.sh:873: get_num_active_clean: grep -v '^$' .../qa/workunits/ceph-helpers.sh:873: get_num_active_clean: wc -l ../qa/workunits/ceph-helpers.sh:993: wait_for_clean: cur_active_clean=0 ../qa/workunits/ceph-helpers.sh:994: wait_for_clean: test 0 '!=' 0 ../qa/workunits/ceph-helpers.sh:998: wait_for_clean: sleep 1 .../qa/workunits/ceph-helpers.sh:999: wait_for_clean: expr 81 + 1 ../qa/workunits/ceph-helpers.sh:999: wait_for_clean: timer=82 ../qa/workunits/ceph-helpers.sh:985: wait_for_clean: is_clean .../qa/workunits/ceph-helpers.sh:952: is_clean: get_num_pgs .../qa/workunits/ceph-helpers.sh:898: get_num_pgs: ceph --format xml status .../qa/workunits/ceph-helpers.sh:899: get_num_pgs: xmlstarlet sel -t -m //pgmap/num_pgs -v . ../qa/workunits/ceph-helpers.sh:952: is_clean: num_pgs=4 ../qa/workunits/ceph-helpers.sh:953: is_clean: test 4 '!=' 0 .../qa/workunits/ceph-helpers.sh:954: is_clean: get_num_active_clean .../qa/workunits/ceph-helpers.sh:863: get_num_active_clean: local 'expression=(' .../qa/workunits/ceph-helpers.sh:864: get_num_active_clean: expression+='contains(.,'\''active'\'') and ' .../qa/workunits/ceph-helpers.sh:865: get_num_active_clean: expression+='contains(.,'\''clean'\'') and ' .../qa/workunits/ceph-helpers.sh:866: get_num_active_clean: expression+='not(contains(.,'\''stale'\''))' .../qa/workunits/ceph-helpers.sh:867: get_num_active_clean: expression+=')' .../qa/workunits/ceph-helpers.sh:871: get_num_active_clean: ceph --format xml pg dump pgs .../qa/workunits/ceph-helpers.sh:872: get_num_active_clean: xmlstarlet sel -t -m '//pg_stat/state[(contains(.,'\''active'\'') and contains(.,'\''clean'\'') and not(contains(.,'\''stale'\'')))]' -v . -n .../qa/workunits/ceph-helpers.sh:873: get_num_active_clean: wc -l .../qa/workunits/ceph-helpers.sh:873: get_num_active_clean: grep -v '^$' ../qa/workunits/ceph-helpers.sh:954: is_clean: test 0 = 4 ../qa/workunits/ceph-helpers.sh:954: is_clean: return 1 ../qa/workunits/ceph-helpers.sh:986: wait_for_clean: get_is_making_recovery_progress .../qa/workunits/ceph-helpers.sh:839: get_is_making_recovery_progress: ceph --format xml status .../qa/workunits/ceph-helpers.sh:839: get_is_making_recovery_progress: xmlstarlet sel -t -m //pgmap/recovering_keys_per_sec -v . -o ' ' -t -m //pgmap/recovering_bytes_per_sec -v . -o ' ' -t -m //pgmap/recovering_objects_per_sec -v . ../qa/workunits/ceph-helpers.sh:839: get_is_making_recovery_progress: local progress= ../qa/workunits/ceph-helpers.sh:840: get_is_making_recovery_progress: test -n '' ../qa/workunits/ceph-helpers.sh:988: wait_for_clean: (( timer >= 120 )) .../qa/workunits/ceph-helpers.sh:993: wait_for_clean: get_num_active_clean .../qa/workunits/ceph-helpers.sh:863: get_num_active_clean: local 'expression=(' .../qa/workunits/ceph-helpers.sh:864: get_num_active_clean: expression+='contains(.,'\''active'\'') and ' .../qa/workunits/ceph-helpers.sh:865: get_num_active_clean: expression+='contains(.,'\''clean'\'') and ' .../qa/workunits/ceph-helpers.sh:866: get_num_active_clean: expression+='not(contains(.,'\''stale'\''))' .../qa/workunits/ceph-helpers.sh:867: get_num_active_clean: expression+=')' .../qa/workunits/ceph-helpers.sh:871: get_num_active_clean: ceph --format xml pg dump pgs .../qa/workunits/ceph-helpers.sh:872: get_num_active_clean: xmlstarlet sel -t -m '//pg_stat/state[(contains(.,'\''active'\'') and contains(.,'\''clean'\'') and not(contains(.,'\''stale'\'')))]' -v . -n .../qa/workunits/ceph-helpers.sh:873: get_num_active_clean: grep -v '^$' .../qa/workunits/ceph-helpers.sh:873: get_num_active_clean: wc -l ../qa/workunits/ceph-helpers.sh:993: wait_for_clean: cur_active_clean=0 ../qa/workunits/ceph-helpers.sh:994: wait_for_clean: test 0 '!=' 0 ../qa/workunits/ceph-helpers.sh:998: wait_for_clean: sleep 1 .../qa/workunits/ceph-helpers.sh:999: wait_for_clean: expr 82 + 1 ../qa/workunits/ceph-helpers.sh:999: wait_for_clean: timer=83 ../qa/workunits/ceph-helpers.sh:985: wait_for_clean: is_clean .../qa/workunits/ceph-helpers.sh:952: is_clean: get_num_pgs .../qa/workunits/ceph-helpers.sh:898: get_num_pgs: ceph --format xml status .../qa/workunits/ceph-helpers.sh:899: get_num_pgs: xmlstarlet sel -t -m //pgmap/num_pgs -v . ../qa/workunits/ceph-helpers.sh:952: is_clean: num_pgs=4 ../qa/workunits/ceph-helpers.sh:953: is_clean: test 4 '!=' 0 .../qa/workunits/ceph-helpers.sh:954: is_clean: get_num_active_clean .../qa/workunits/ceph-helpers.sh:863: get_num_active_clean: local 'expression=(' .../qa/workunits/ceph-helpers.sh:864: get_num_active_clean: expression+='contains(.,'\''active'\'') and ' .../qa/workunits/ceph-helpers.sh:865: get_num_active_clean: expression+='contains(.,'\''clean'\'') and ' .../qa/workunits/ceph-helpers.sh:866: get_num_active_clean: expression+='not(contains(.,'\''stale'\''))' .../qa/workunits/ceph-helpers.sh:867: get_num_active_clean: expression+=')' .../qa/workunits/ceph-helpers.sh:871: get_num_active_clean: ceph --format xml pg dump pgs .../qa/workunits/ceph-helpers.sh:872: get_num_active_clean: xmlstarlet sel -t -m '//pg_stat/state[(contains(.,'\''active'\'') and contains(.,'\''clean'\'') and not(contains(.,'\''stale'\'')))]' -v . -n .../qa/workunits/ceph-helpers.sh:873: get_num_active_clean: grep -v '^$' .../qa/workunits/ceph-helpers.sh:873: get_num_active_clean: wc -l ../qa/workunits/ceph-helpers.sh:954: is_clean: test 0 = 4 ../qa/workunits/ceph-helpers.sh:954: is_clean: return 1 ../qa/workunits/ceph-helpers.sh:986: wait_for_clean: get_is_making_recovery_progress .../qa/workunits/ceph-helpers.sh:839: get_is_making_recovery_progress: ceph --format xml status .../qa/workunits/ceph-helpers.sh:839: get_is_making_recovery_progress: xmlstarlet sel -t -m //pgmap/recovering_keys_per_sec -v . -o ' ' -t -m //pgmap/recovering_bytes_per_sec -v . -o ' ' -t -m //pgmap/recovering_objects_per_sec -v . ../qa/workunits/ceph-helpers.sh:839: get_is_making_recovery_progress: local progress= ../qa/workunits/ceph-helpers.sh:840: get_is_making_recovery_progress: test -n '' ../qa/workunits/ceph-helpers.sh:988: wait_for_clean: (( timer >= 120 )) .../qa/workunits/ceph-helpers.sh:993: wait_for_clean: get_num_active_clean .../qa/workunits/ceph-helpers.sh:863: get_num_active_clean: local 'expression=(' .../qa/workunits/ceph-helpers.sh:864: get_num_active_clean: expression+='contains(.,'\''active'\'') and ' .../qa/workunits/ceph-helpers.sh:865: get_num_active_clean: expression+='contains(.,'\''clean'\'') and ' .../qa/workunits/ceph-helpers.sh:866: get_num_active_clean: expression+='not(contains(.,'\''stale'\''))' .../qa/workunits/ceph-helpers.sh:867: get_num_active_clean: expression+=')' .../qa/workunits/ceph-helpers.sh:871: get_num_active_clean: ceph --format xml pg dump pgs .../qa/workunits/ceph-helpers.sh:872: get_num_active_clean: xmlstarlet sel -t -m '//pg_stat/state[(contains(.,'\''active'\'') and contains(.,'\''clean'\'') and not(contains(.,'\''stale'\'')))]' -v . -n .../qa/workunits/ceph-helpers.sh:873: get_num_active_clean: grep -v '^$' .../qa/workunits/ceph-helpers.sh:873: get_num_active_clean: wc -l ../qa/workunits/ceph-helpers.sh:993: wait_for_clean: cur_active_clean=0 ../qa/workunits/ceph-helpers.sh:994: wait_for_clean: test 0 '!=' 0 ../qa/workunits/ceph-helpers.sh:998: wait_for_clean: sleep 1 .../qa/workunits/ceph-helpers.sh:999: wait_for_clean: expr 83 + 1 ../qa/workunits/ceph-helpers.sh:999: wait_for_clean: timer=84 ../qa/workunits/ceph-helpers.sh:985: wait_for_clean: is_clean .../qa/workunits/ceph-helpers.sh:952: is_clean: get_num_pgs .../qa/workunits/ceph-helpers.sh:898: get_num_pgs: ceph --format xml status .../qa/workunits/ceph-helpers.sh:899: get_num_pgs: xmlstarlet sel -t -m //pgmap/num_pgs -v . ../qa/workunits/ceph-helpers.sh:952: is_clean: num_pgs=4 ../qa/workunits/ceph-helpers.sh:953: is_clean: test 4 '!=' 0 .../qa/workunits/ceph-helpers.sh:954: is_clean: get_num_active_clean .../qa/workunits/ceph-helpers.sh:863: get_num_active_clean: local 'expression=(' .../qa/workunits/ceph-helpers.sh:864: get_num_active_clean: expression+='contains(.,'\''active'\'') and ' .../qa/workunits/ceph-helpers.sh:865: get_num_active_clean: expression+='contains(.,'\''clean'\'') and ' .../qa/workunits/ceph-helpers.sh:866: get_num_active_clean: expression+='not(contains(.,'\''stale'\''))' .../qa/workunits/ceph-helpers.sh:867: get_num_active_clean: expression+=')' .../qa/workunits/ceph-helpers.sh:871: get_num_active_clean: ceph --format xml pg dump pgs .../qa/workunits/ceph-helpers.sh:872: get_num_active_clean: xmlstarlet sel -t -m '//pg_stat/state[(contains(.,'\''active'\'') and contains(.,'\''clean'\'') and not(contains(.,'\''stale'\'')))]' -v . -n .../qa/workunits/ceph-helpers.sh:873: get_num_active_clean: grep -v '^$' .../qa/workunits/ceph-helpers.sh:873: get_num_active_clean: wc -l ../qa/workunits/ceph-helpers.sh:954: is_clean: test 0 = 4 ../qa/workunits/ceph-helpers.sh:954: is_clean: return 1 ../qa/workunits/ceph-helpers.sh:986: wait_for_clean: get_is_making_recovery_progress .../qa/workunits/ceph-helpers.sh:839: get_is_making_recovery_progress: ceph --format xml status .../qa/workunits/ceph-helpers.sh:839: get_is_making_recovery_progress: xmlstarlet sel -t -m //pgmap/recovering_keys_per_sec -v . -o ' ' -t -m //pgmap/recovering_bytes_per_sec -v . -o ' ' -t -m //pgmap/recovering_objects_per_sec -v . ../qa/workunits/ceph-helpers.sh:839: get_is_making_recovery_progress: local progress= ../qa/workunits/ceph-helpers.sh:840: get_is_making_recovery_progress: test -n '' ../qa/workunits/ceph-helpers.sh:988: wait_for_clean: (( timer >= 120 )) .../qa/workunits/ceph-helpers.sh:993: wait_for_clean: get_num_active_clean .../qa/workunits/ceph-helpers.sh:863: get_num_active_clean: local 'expression=(' .../qa/workunits/ceph-helpers.sh:864: get_num_active_clean: expression+='contains(.,'\''active'\'') and ' .../qa/workunits/ceph-helpers.sh:865: get_num_active_clean: expression+='contains(.,'\''clean'\'') and ' .../qa/workunits/ceph-helpers.sh:866: get_num_active_clean: expression+='not(contains(.,'\''stale'\''))' .../qa/workunits/ceph-helpers.sh:867: get_num_active_clean: expression+=')' .../qa/workunits/ceph-helpers.sh:871: get_num_active_clean: ceph --format xml pg dump pgs .../qa/workunits/ceph-helpers.sh:872: get_num_active_clean: xmlstarlet sel -t -m '//pg_stat/state[(contains(.,'\''active'\'') and contains(.,'\''clean'\'') and not(contains(.,'\''stale'\'')))]' -v . -n .../qa/workunits/ceph-helpers.sh:873: get_num_active_clean: grep -v '^$' .../qa/workunits/ceph-helpers.sh:873: get_num_active_clean: wc -l ../qa/workunits/ceph-helpers.sh:993: wait_for_clean: cur_active_clean=0 ../qa/workunits/ceph-helpers.sh:994: wait_for_clean: test 0 '!=' 0 ../qa/workunits/ceph-helpers.sh:998: wait_for_clean: sleep 1 .../qa/workunits/ceph-helpers.sh:999: wait_for_clean: expr 84 + 1 ../qa/workunits/ceph-helpers.sh:999: wait_for_clean: timer=85 ../qa/workunits/ceph-helpers.sh:985: wait_for_clean: is_clean .../qa/workunits/ceph-helpers.sh:952: is_clean: get_num_pgs .../qa/workunits/ceph-helpers.sh:898: get_num_pgs: ceph --format xml status .../qa/workunits/ceph-helpers.sh:899: get_num_pgs: xmlstarlet sel -t -m //pgmap/num_pgs -v . ../qa/workunits/ceph-helpers.sh:952: is_clean: num_pgs=4 ../qa/workunits/ceph-helpers.sh:953: is_clean: test 4 '!=' 0 .../qa/workunits/ceph-helpers.sh:954: is_clean: get_num_active_clean .../qa/workunits/ceph-helpers.sh:863: get_num_active_clean: local 'expression=(' .../qa/workunits/ceph-helpers.sh:864: get_num_active_clean: expression+='contains(.,'\''active'\'') and ' .../qa/workunits/ceph-helpers.sh:865: get_num_active_clean: expression+='contains(.,'\''clean'\'') and ' .../qa/workunits/ceph-helpers.sh:866: get_num_active_clean: expression+='not(contains(.,'\''stale'\''))' .../qa/workunits/ceph-helpers.sh:867: get_num_active_clean: expression+=')' .../qa/workunits/ceph-helpers.sh:871: get_num_active_clean: ceph --format xml pg dump pgs .../qa/workunits/ceph-helpers.sh:872: get_num_active_clean: xmlstarlet sel -t -m '//pg_stat/state[(contains(.,'\''active'\'') and contains(.,'\''clean'\'') and not(contains(.,'\''stale'\'')))]' -v . -n .../qa/workunits/ceph-helpers.sh:873: get_num_active_clean: grep -v '^$' .../qa/workunits/ceph-helpers.sh:873: get_num_active_clean: wc -l ../qa/workunits/ceph-helpers.sh:954: is_clean: test 0 = 4 ../qa/workunits/ceph-helpers.sh:954: is_clean: return 1 ../qa/workunits/ceph-helpers.sh:986: wait_for_clean: get_is_making_recovery_progress .../qa/workunits/ceph-helpers.sh:839: get_is_making_recovery_progress: ceph --format xml status .../qa/workunits/ceph-helpers.sh:839: get_is_making_recovery_progress: xmlstarlet sel -t -m //pgmap/recovering_keys_per_sec -v . -o ' ' -t -m //pgmap/recovering_bytes_per_sec -v . -o ' ' -t -m //pgmap/recovering_objects_per_sec -v . ../qa/workunits/ceph-helpers.sh:839: get_is_making_recovery_progress: local progress= ../qa/workunits/ceph-helpers.sh:840: get_is_making_recovery_progress: test -n '' ../qa/workunits/ceph-helpers.sh:988: wait_for_clean: (( timer >= 120 )) .../qa/workunits/ceph-helpers.sh:993: wait_for_clean: get_num_active_clean .../qa/workunits/ceph-helpers.sh:863: get_num_active_clean: local 'expression=(' .../qa/workunits/ceph-helpers.sh:864: get_num_active_clean: expression+='contains(.,'\''active'\'') and ' .../qa/workunits/ceph-helpers.sh:865: get_num_active_clean: expression+='contains(.,'\''clean'\'') and ' .../qa/workunits/ceph-helpers.sh:866: get_num_active_clean: expression+='not(contains(.,'\''stale'\''))' .../qa/workunits/ceph-helpers.sh:867: get_num_active_clean: expression+=')' .../qa/workunits/ceph-helpers.sh:871: get_num_active_clean: ceph --format xml pg dump pgs .../qa/workunits/ceph-helpers.sh:872: get_num_active_clean: xmlstarlet sel -t -m '//pg_stat/state[(contains(.,'\''active'\'') and contains(.,'\''clean'\'') and not(contains(.,'\''stale'\'')))]' -v . -n .../qa/workunits/ceph-helpers.sh:873: get_num_active_clean: grep -v '^$' .../qa/workunits/ceph-helpers.sh:873: get_num_active_clean: wc -l ../qa/workunits/ceph-helpers.sh:993: wait_for_clean: cur_active_clean=0 ../qa/workunits/ceph-helpers.sh:994: wait_for_clean: test 0 '!=' 0 ../qa/workunits/ceph-helpers.sh:998: wait_for_clean: sleep 1 .../qa/workunits/ceph-helpers.sh:999: wait_for_clean: expr 85 + 1 ../qa/workunits/ceph-helpers.sh:999: wait_for_clean: timer=86 ../qa/workunits/ceph-helpers.sh:985: wait_for_clean: is_clean .../qa/workunits/ceph-helpers.sh:952: is_clean: get_num_pgs .../qa/workunits/ceph-helpers.sh:898: get_num_pgs: ceph --format xml status .../qa/workunits/ceph-helpers.sh:899: get_num_pgs: xmlstarlet sel -t -m //pgmap/num_pgs -v . ../qa/workunits/ceph-helpers.sh:952: is_clean: num_pgs=4 ../qa/workunits/ceph-helpers.sh:953: is_clean: test 4 '!=' 0 .../qa/workunits/ceph-helpers.sh:954: is_clean: get_num_active_clean .../qa/workunits/ceph-helpers.sh:863: get_num_active_clean: local 'expression=(' .../qa/workunits/ceph-helpers.sh:864: get_num_active_clean: expression+='contains(.,'\''active'\'') and ' .../qa/workunits/ceph-helpers.sh:865: get_num_active_clean: expression+='contains(.,'\''clean'\'') and ' .../qa/workunits/ceph-helpers.sh:866: get_num_active_clean: expression+='not(contains(.,'\''stale'\''))' .../qa/workunits/ceph-helpers.sh:867: get_num_active_clean: expression+=')' .../qa/workunits/ceph-helpers.sh:871: get_num_active_clean: ceph --format xml pg dump pgs .../qa/workunits/ceph-helpers.sh:872: get_num_active_clean: xmlstarlet sel -t -m '//pg_stat/state[(contains(.,'\''active'\'') and contains(.,'\''clean'\'') and not(contains(.,'\''stale'\'')))]' -v . -n .../qa/workunits/ceph-helpers.sh:873: get_num_active_clean: grep -v '^$' .../qa/workunits/ceph-helpers.sh:873: get_num_active_clean: wc -l ../qa/workunits/ceph-helpers.sh:954: is_clean: test 0 = 4 ../qa/workunits/ceph-helpers.sh:954: is_clean: return 1 ../qa/workunits/ceph-helpers.sh:986: wait_for_clean: get_is_making_recovery_progress .../qa/workunits/ceph-helpers.sh:839: get_is_making_recovery_progress: ceph --format xml status .../qa/workunits/ceph-helpers.sh:839: get_is_making_recovery_progress: xmlstarlet sel -t -m //pgmap/recovering_keys_per_sec -v . -o ' ' -t -m //pgmap/recovering_bytes_per_sec -v . -o ' ' -t -m //pgmap/recovering_objects_per_sec -v . ../qa/workunits/ceph-helpers.sh:839: get_is_making_recovery_progress: local progress= ../qa/workunits/ceph-helpers.sh:840: get_is_making_recovery_progress: test -n '' ../qa/workunits/ceph-helpers.sh:988: wait_for_clean: (( timer >= 120 )) .../qa/workunits/ceph-helpers.sh:993: wait_for_clean: get_num_active_clean .../qa/workunits/ceph-helpers.sh:863: get_num_active_clean: local 'expression=(' .../qa/workunits/ceph-helpers.sh:864: get_num_active_clean: expression+='contains(.,'\''active'\'') and ' .../qa/workunits/ceph-helpers.sh:865: get_num_active_clean: expression+='contains(.,'\''clean'\'') and ' .../qa/workunits/ceph-helpers.sh:866: get_num_active_clean: expression+='not(contains(.,'\''stale'\''))' .../qa/workunits/ceph-helpers.sh:867: get_num_active_clean: expression+=')' .../qa/workunits/ceph-helpers.sh:871: get_num_active_clean: ceph --format xml pg dump pgs .../qa/workunits/ceph-helpers.sh:872: get_num_active_clean: xmlstarlet sel -t -m '//pg_stat/state[(contains(.,'\''active'\'') and contains(.,'\''clean'\'') and not(contains(.,'\''stale'\'')))]' -v . -n .../qa/workunits/ceph-helpers.sh:873: get_num_active_clean: grep -v '^$' .../qa/workunits/ceph-helpers.sh:873: get_num_active_clean: wc -l ../qa/workunits/ceph-helpers.sh:993: wait_for_clean: cur_active_clean=0 ../qa/workunits/ceph-helpers.sh:994: wait_for_clean: test 0 '!=' 0 ../qa/workunits/ceph-helpers.sh:998: wait_for_clean: sleep 1 .../qa/workunits/ceph-helpers.sh:999: wait_for_clean: expr 86 + 1 ../qa/workunits/ceph-helpers.sh:999: wait_for_clean: timer=87 ../qa/workunits/ceph-helpers.sh:985: wait_for_clean: is_clean .../qa/workunits/ceph-helpers.sh:952: is_clean: get_num_pgs .../qa/workunits/ceph-helpers.sh:898: get_num_pgs: ceph --format xml status .../qa/workunits/ceph-helpers.sh:899: get_num_pgs: xmlstarlet sel -t -m //pgmap/num_pgs -v . ../qa/workunits/ceph-helpers.sh:952: is_clean: num_pgs=4 ../qa/workunits/ceph-helpers.sh:953: is_clean: test 4 '!=' 0 .../qa/workunits/ceph-helpers.sh:954: is_clean: get_num_active_clean .../qa/workunits/ceph-helpers.sh:863: get_num_active_clean: local 'expression=(' .../qa/workunits/ceph-helpers.sh:864: get_num_active_clean: expression+='contains(.,'\''active'\'') and ' .../qa/workunits/ceph-helpers.sh:865: get_num_active_clean: expression+='contains(.,'\''clean'\'') and ' .../qa/workunits/ceph-helpers.sh:866: get_num_active_clean: expression+='not(contains(.,'\''stale'\''))' .../qa/workunits/ceph-helpers.sh:867: get_num_active_clean: expression+=')' .../qa/workunits/ceph-helpers.sh:871: get_num_active_clean: ceph --format xml pg dump pgs .../qa/workunits/ceph-helpers.sh:872: get_num_active_clean: xmlstarlet sel -t -m '//pg_stat/state[(contains(.,'\''active'\'') and contains(.,'\''clean'\'') and not(contains(.,'\''stale'\'')))]' -v . -n .../qa/workunits/ceph-helpers.sh:873: get_num_active_clean: wc -l .../qa/workunits/ceph-helpers.sh:873: get_num_active_clean: grep -v '^$' ../qa/workunits/ceph-helpers.sh:954: is_clean: test 0 = 4 ../qa/workunits/ceph-helpers.sh:954: is_clean: return 1 ../qa/workunits/ceph-helpers.sh:986: wait_for_clean: get_is_making_recovery_progress .../qa/workunits/ceph-helpers.sh:839: get_is_making_recovery_progress: ceph --format xml status .../qa/workunits/ceph-helpers.sh:839: get_is_making_recovery_progress: xmlstarlet sel -t -m //pgmap/recovering_keys_per_sec -v . -o ' ' -t -m //pgmap/recovering_bytes_per_sec -v . -o ' ' -t -m //pgmap/recovering_objects_per_sec -v . ../qa/workunits/ceph-helpers.sh:839: get_is_making_recovery_progress: local progress= ../qa/workunits/ceph-helpers.sh:840: get_is_making_recovery_progress: test -n '' ../qa/workunits/ceph-helpers.sh:988: wait_for_clean: (( timer >= 120 )) .../qa/workunits/ceph-helpers.sh:993: wait_for_clean: get_num_active_clean .../qa/workunits/ceph-helpers.sh:863: get_num_active_clean: local 'expression=(' .../qa/workunits/ceph-helpers.sh:864: get_num_active_clean: expression+='contains(.,'\''active'\'') and ' .../qa/workunits/ceph-helpers.sh:865: get_num_active_clean: expression+='contains(.,'\''clean'\'') and ' .../qa/workunits/ceph-helpers.sh:866: get_num_active_clean: expression+='not(contains(.,'\''stale'\''))' .../qa/workunits/ceph-helpers.sh:867: get_num_active_clean: expression+=')' .../qa/workunits/ceph-helpers.sh:871: get_num_active_clean: ceph --format xml pg dump pgs .../qa/workunits/ceph-helpers.sh:872: get_num_active_clean: xmlstarlet sel -t -m '//pg_stat/state[(contains(.,'\''active'\'') and contains(.,'\''clean'\'') and not(contains(.,'\''stale'\'')))]' -v . -n .../qa/workunits/ceph-helpers.sh:873: get_num_active_clean: grep -v '^$' .../qa/workunits/ceph-helpers.sh:873: get_num_active_clean: wc -l ../qa/workunits/ceph-helpers.sh:993: wait_for_clean: cur_active_clean=0 ../qa/workunits/ceph-helpers.sh:994: wait_for_clean: test 0 '!=' 0 ../qa/workunits/ceph-helpers.sh:998: wait_for_clean: sleep 1 .../qa/workunits/ceph-helpers.sh:999: wait_for_clean: expr 87 + 1 ../qa/workunits/ceph-helpers.sh:999: wait_for_clean: timer=88 ../qa/workunits/ceph-helpers.sh:985: wait_for_clean: is_clean .../qa/workunits/ceph-helpers.sh:952: is_clean: get_num_pgs .../qa/workunits/ceph-helpers.sh:898: get_num_pgs: ceph --format xml status .../qa/workunits/ceph-helpers.sh:899: get_num_pgs: xmlstarlet sel -t -m //pgmap/num_pgs -v . ../qa/workunits/ceph-helpers.sh:952: is_clean: num_pgs=4 ../qa/workunits/ceph-helpers.sh:953: is_clean: test 4 '!=' 0 .../qa/workunits/ceph-helpers.sh:954: is_clean: get_num_active_clean .../qa/workunits/ceph-helpers.sh:863: get_num_active_clean: local 'expression=(' .../qa/workunits/ceph-helpers.sh:864: get_num_active_clean: expression+='contains(.,'\''active'\'') and ' .../qa/workunits/ceph-helpers.sh:865: get_num_active_clean: expression+='contains(.,'\''clean'\'') and ' .../qa/workunits/ceph-helpers.sh:866: get_num_active_clean: expression+='not(contains(.,'\''stale'\''))' .../qa/workunits/ceph-helpers.sh:867: get_num_active_clean: expression+=')' .../qa/workunits/ceph-helpers.sh:871: get_num_active_clean: ceph --format xml pg dump pgs .../qa/workunits/ceph-helpers.sh:872: get_num_active_clean: xmlstarlet sel -t -m '//pg_stat/state[(contains(.,'\''active'\'') and contains(.,'\''clean'\'') and not(contains(.,'\''stale'\'')))]' -v . -n .../qa/workunits/ceph-helpers.sh:873: get_num_active_clean: grep -v '^$' .../qa/workunits/ceph-helpers.sh:873: get_num_active_clean: wc -l ../qa/workunits/ceph-helpers.sh:954: is_clean: test 0 = 4 ../qa/workunits/ceph-helpers.sh:954: is_clean: return 1 ../qa/workunits/ceph-helpers.sh:986: wait_for_clean: get_is_making_recovery_progress .../qa/workunits/ceph-helpers.sh:839: get_is_making_recovery_progress: ceph --format xml status .../qa/workunits/ceph-helpers.sh:839: get_is_making_recovery_progress: xmlstarlet sel -t -m //pgmap/recovering_keys_per_sec -v . -o ' ' -t -m //pgmap/recovering_bytes_per_sec -v . -o ' ' -t -m //pgmap/recovering_objects_per_sec -v . ../qa/workunits/ceph-helpers.sh:839: get_is_making_recovery_progress: local progress= ../qa/workunits/ceph-helpers.sh:840: get_is_making_recovery_progress: test -n '' ../qa/workunits/ceph-helpers.sh:988: wait_for_clean: (( timer >= 120 )) .../qa/workunits/ceph-helpers.sh:993: wait_for_clean: get_num_active_clean .../qa/workunits/ceph-helpers.sh:863: get_num_active_clean: local 'expression=(' .../qa/workunits/ceph-helpers.sh:864: get_num_active_clean: expression+='contains(.,'\''active'\'') and ' .../qa/workunits/ceph-helpers.sh:865: get_num_active_clean: expression+='contains(.,'\''clean'\'') and ' .../qa/workunits/ceph-helpers.sh:866: get_num_active_clean: expression+='not(contains(.,'\''stale'\''))' .../qa/workunits/ceph-helpers.sh:867: get_num_active_clean: expression+=')' .../qa/workunits/ceph-helpers.sh:871: get_num_active_clean: ceph --format xml pg dump pgs .../qa/workunits/ceph-helpers.sh:872: get_num_active_clean: xmlstarlet sel -t -m '//pg_stat/state[(contains(.,'\''active'\'') and contains(.,'\''clean'\'') and not(contains(.,'\''stale'\'')))]' -v . -n .../qa/workunits/ceph-helpers.sh:873: get_num_active_clean: grep -v '^$' .../qa/workunits/ceph-helpers.sh:873: get_num_active_clean: wc -l ../qa/workunits/ceph-helpers.sh:993: wait_for_clean: cur_active_clean=0 ../qa/workunits/ceph-helpers.sh:994: wait_for_clean: test 0 '!=' 0 ../qa/workunits/ceph-helpers.sh:998: wait_for_clean: sleep 1 .../qa/workunits/ceph-helpers.sh:999: wait_for_clean: expr 88 + 1 ../qa/workunits/ceph-helpers.sh:999: wait_for_clean: timer=89 ../qa/workunits/ceph-helpers.sh:985: wait_for_clean: is_clean .../qa/workunits/ceph-helpers.sh:952: is_clean: get_num_pgs .../qa/workunits/ceph-helpers.sh:898: get_num_pgs: ceph --format xml status .../qa/workunits/ceph-helpers.sh:899: get_num_pgs: xmlstarlet sel -t -m //pgmap/num_pgs -v . ../qa/workunits/ceph-helpers.sh:952: is_clean: num_pgs=4 ../qa/workunits/ceph-helpers.sh:953: is_clean: test 4 '!=' 0 .../qa/workunits/ceph-helpers.sh:954: is_clean: get_num_active_clean .../qa/workunits/ceph-helpers.sh:863: get_num_active_clean: local 'expression=(' .../qa/workunits/ceph-helpers.sh:864: get_num_active_clean: expression+='contains(.,'\''active'\'') and ' .../qa/workunits/ceph-helpers.sh:865: get_num_active_clean: expression+='contains(.,'\''clean'\'') and ' .../qa/workunits/ceph-helpers.sh:866: get_num_active_clean: expression+='not(contains(.,'\''stale'\''))' .../qa/workunits/ceph-helpers.sh:867: get_num_active_clean: expression+=')' .../qa/workunits/ceph-helpers.sh:871: get_num_active_clean: ceph --format xml pg dump pgs .../qa/workunits/ceph-helpers.sh:872: get_num_active_clean: xmlstarlet sel -t -m '//pg_stat/state[(contains(.,'\''active'\'') and contains(.,'\''clean'\'') and not(contains(.,'\''stale'\'')))]' -v . -n .../qa/workunits/ceph-helpers.sh:873: get_num_active_clean: grep -v '^$' .../qa/workunits/ceph-helpers.sh:873: get_num_active_clean: wc -l ../qa/workunits/ceph-helpers.sh:954: is_clean: test 0 = 4 ../qa/workunits/ceph-helpers.sh:954: is_clean: return 1 ../qa/workunits/ceph-helpers.sh:986: wait_for_clean: get_is_making_recovery_progress .../qa/workunits/ceph-helpers.sh:839: get_is_making_recovery_progress: ceph --format xml status .../qa/workunits/ceph-helpers.sh:839: get_is_making_recovery_progress: xmlstarlet sel -t -m //pgmap/recovering_keys_per_sec -v . -o ' ' -t -m //pgmap/recovering_bytes_per_sec -v . -o ' ' -t -m //pgmap/recovering_objects_per_sec -v . ../qa/workunits/ceph-helpers.sh:839: get_is_making_recovery_progress: local progress= ../qa/workunits/ceph-helpers.sh:840: get_is_making_recovery_progress: test -n '' ../qa/workunits/ceph-helpers.sh:988: wait_for_clean: (( timer >= 120 )) .../qa/workunits/ceph-helpers.sh:993: wait_for_clean: get_num_active_clean .../qa/workunits/ceph-helpers.sh:863: get_num_active_clean: local 'expression=(' .../qa/workunits/ceph-helpers.sh:864: get_num_active_clean: expression+='contains(.,'\''active'\'') and ' .../qa/workunits/ceph-helpers.sh:865: get_num_active_clean: expression+='contains(.,'\''clean'\'') and ' .../qa/workunits/ceph-helpers.sh:866: get_num_active_clean: expression+='not(contains(.,'\''stale'\''))' .../qa/workunits/ceph-helpers.sh:867: get_num_active_clean: expression+=')' .../qa/workunits/ceph-helpers.sh:871: get_num_active_clean: ceph --format xml pg dump pgs .../qa/workunits/ceph-helpers.sh:872: get_num_active_clean: xmlstarlet sel -t -m '//pg_stat/state[(contains(.,'\''active'\'') and contains(.,'\''clean'\'') and not(contains(.,'\''stale'\'')))]' -v . -n .../qa/workunits/ceph-helpers.sh:873: get_num_active_clean: grep -v '^$' .../qa/workunits/ceph-helpers.sh:873: get_num_active_clean: wc -l ../qa/workunits/ceph-helpers.sh:993: wait_for_clean: cur_active_clean=0 ../qa/workunits/ceph-helpers.sh:994: wait_for_clean: test 0 '!=' 0 ../qa/workunits/ceph-helpers.sh:998: wait_for_clean: sleep 1 .../qa/workunits/ceph-helpers.sh:999: wait_for_clean: expr 89 + 1 ../qa/workunits/ceph-helpers.sh:999: wait_for_clean: timer=90 ../qa/workunits/ceph-helpers.sh:985: wait_for_clean: is_clean .../qa/workunits/ceph-helpers.sh:952: is_clean: get_num_pgs .../qa/workunits/ceph-helpers.sh:898: get_num_pgs: ceph --format xml status .../qa/workunits/ceph-helpers.sh:899: get_num_pgs: xmlstarlet sel -t -m //pgmap/num_pgs -v . ../qa/workunits/ceph-helpers.sh:952: is_clean: num_pgs=4 ../qa/workunits/ceph-helpers.sh:953: is_clean: test 4 '!=' 0 .../qa/workunits/ceph-helpers.sh:954: is_clean: get_num_active_clean .../qa/workunits/ceph-helpers.sh:863: get_num_active_clean: local 'expression=(' .../qa/workunits/ceph-helpers.sh:864: get_num_active_clean: expression+='contains(.,'\''active'\'') and ' .../qa/workunits/ceph-helpers.sh:865: get_num_active_clean: expression+='contains(.,'\''clean'\'') and ' .../qa/workunits/ceph-helpers.sh:866: get_num_active_clean: expression+='not(contains(.,'\''stale'\''))' .../qa/workunits/ceph-helpers.sh:867: get_num_active_clean: expression+=')' .../qa/workunits/ceph-helpers.sh:871: get_num_active_clean: ceph --format xml pg dump pgs .../qa/workunits/ceph-helpers.sh:872: get_num_active_clean: xmlstarlet sel -t -m '//pg_stat/state[(contains(.,'\''active'\'') and contains(.,'\''clean'\'') and not(contains(.,'\''stale'\'')))]' -v . -n .../qa/workunits/ceph-helpers.sh:873: get_num_active_clean: grep -v '^$' .../qa/workunits/ceph-helpers.sh:873: get_num_active_clean: wc -l ../qa/workunits/ceph-helpers.sh:954: is_clean: test 0 = 4 ../qa/workunits/ceph-helpers.sh:954: is_clean: return 1 ../qa/workunits/ceph-helpers.sh:986: wait_for_clean: get_is_making_recovery_progress .../qa/workunits/ceph-helpers.sh:839: get_is_making_recovery_progress: ceph --format xml status .../qa/workunits/ceph-helpers.sh:839: get_is_making_recovery_progress: xmlstarlet sel -t -m //pgmap/recovering_keys_per_sec -v . -o ' ' -t -m //pgmap/recovering_bytes_per_sec -v . -o ' ' -t -m //pgmap/recovering_objects_per_sec -v . ../qa/workunits/ceph-helpers.sh:839: get_is_making_recovery_progress: local progress= ../qa/workunits/ceph-helpers.sh:840: get_is_making_recovery_progress: test -n '' ../qa/workunits/ceph-helpers.sh:988: wait_for_clean: (( timer >= 120 )) .../qa/workunits/ceph-helpers.sh:993: wait_for_clean: get_num_active_clean .../qa/workunits/ceph-helpers.sh:863: get_num_active_clean: local 'expression=(' .../qa/workunits/ceph-helpers.sh:864: get_num_active_clean: expression+='contains(.,'\''active'\'') and ' .../qa/workunits/ceph-helpers.sh:865: get_num_active_clean: expression+='contains(.,'\''clean'\'') and ' .../qa/workunits/ceph-helpers.sh:866: get_num_active_clean: expression+='not(contains(.,'\''stale'\''))' .../qa/workunits/ceph-helpers.sh:867: get_num_active_clean: expression+=')' .../qa/workunits/ceph-helpers.sh:871: get_num_active_clean: ceph --format xml pg dump pgs .../qa/workunits/ceph-helpers.sh:872: get_num_active_clean: xmlstarlet sel -t -m '//pg_stat/state[(contains(.,'\''active'\'') and contains(.,'\''clean'\'') and not(contains(.,'\''stale'\'')))]' -v . -n .../qa/workunits/ceph-helpers.sh:873: get_num_active_clean: grep -v '^$' .../qa/workunits/ceph-helpers.sh:873: get_num_active_clean: wc -l ../qa/workunits/ceph-helpers.sh:993: wait_for_clean: cur_active_clean=0 ../qa/workunits/ceph-helpers.sh:994: wait_for_clean: test 0 '!=' 0 ../qa/workunits/ceph-helpers.sh:998: wait_for_clean: sleep 1 .../qa/workunits/ceph-helpers.sh:999: wait_for_clean: expr 90 + 1 ../qa/workunits/ceph-helpers.sh:999: wait_for_clean: timer=91 ../qa/workunits/ceph-helpers.sh:985: wait_for_clean: is_clean .../qa/workunits/ceph-helpers.sh:952: is_clean: get_num_pgs .../qa/workunits/ceph-helpers.sh:898: get_num_pgs: ceph --format xml status .../qa/workunits/ceph-helpers.sh:899: get_num_pgs: xmlstarlet sel -t -m //pgmap/num_pgs -v . ../qa/workunits/ceph-helpers.sh:952: is_clean: num_pgs=4 ../qa/workunits/ceph-helpers.sh:953: is_clean: test 4 '!=' 0 .../qa/workunits/ceph-helpers.sh:954: is_clean: get_num_active_clean .../qa/workunits/ceph-helpers.sh:863: get_num_active_clean: local 'expression=(' .../qa/workunits/ceph-helpers.sh:864: get_num_active_clean: expression+='contains(.,'\''active'\'') and ' .../qa/workunits/ceph-helpers.sh:865: get_num_active_clean: expression+='contains(.,'\''clean'\'') and ' .../qa/workunits/ceph-helpers.sh:866: get_num_active_clean: expression+='not(contains(.,'\''stale'\''))' .../qa/workunits/ceph-helpers.sh:867: get_num_active_clean: expression+=')' .../qa/workunits/ceph-helpers.sh:871: get_num_active_clean: ceph --format xml pg dump pgs .../qa/workunits/ceph-helpers.sh:872: get_num_active_clean: xmlstarlet sel -t -m '//pg_stat/state[(contains(.,'\''active'\'') and contains(.,'\''clean'\'') and not(contains(.,'\''stale'\'')))]' -v . -n .../qa/workunits/ceph-helpers.sh:873: get_num_active_clean: grep -v '^$' .../qa/workunits/ceph-helpers.sh:873: get_num_active_clean: wc -l ../qa/workunits/ceph-helpers.sh:954: is_clean: test 0 = 4 ../qa/workunits/ceph-helpers.sh:954: is_clean: return 1 ../qa/workunits/ceph-helpers.sh:986: wait_for_clean: get_is_making_recovery_progress .../qa/workunits/ceph-helpers.sh:839: get_is_making_recovery_progress: ceph --format xml status .../qa/workunits/ceph-helpers.sh:839: get_is_making_recovery_progress: xmlstarlet sel -t -m //pgmap/recovering_keys_per_sec -v . -o ' ' -t -m //pgmap/recovering_bytes_per_sec -v . -o ' ' -t -m //pgmap/recovering_objects_per_sec -v . ../qa/workunits/ceph-helpers.sh:839: get_is_making_recovery_progress: local progress= ../qa/workunits/ceph-helpers.sh:840: get_is_making_recovery_progress: test -n '' ../qa/workunits/ceph-helpers.sh:988: wait_for_clean: (( timer >= 120 )) .../qa/workunits/ceph-helpers.sh:993: wait_for_clean: get_num_active_clean .../qa/workunits/ceph-helpers.sh:863: get_num_active_clean: local 'expression=(' .../qa/workunits/ceph-helpers.sh:864: get_num_active_clean: expression+='contains(.,'\''active'\'') and ' .../qa/workunits/ceph-helpers.sh:865: get_num_active_clean: expression+='contains(.,'\''clean'\'') and ' .../qa/workunits/ceph-helpers.sh:866: get_num_active_clean: expression+='not(contains(.,'\''stale'\''))' .../qa/workunits/ceph-helpers.sh:867: get_num_active_clean: expression+=')' .../qa/workunits/ceph-helpers.sh:871: get_num_active_clean: ceph --format xml pg dump pgs .../qa/workunits/ceph-helpers.sh:872: get_num_active_clean: xmlstarlet sel -t -m '//pg_stat/state[(contains(.,'\''active'\'') and contains(.,'\''clean'\'') and not(contains(.,'\''stale'\'')))]' -v . -n .../qa/workunits/ceph-helpers.sh:873: get_num_active_clean: grep -v '^$' .../qa/workunits/ceph-helpers.sh:873: get_num_active_clean: wc -l ../qa/workunits/ceph-helpers.sh:993: wait_for_clean: cur_active_clean=0 ../qa/workunits/ceph-helpers.sh:994: wait_for_clean: test 0 '!=' 0 ../qa/workunits/ceph-helpers.sh:998: wait_for_clean: sleep 1 .../qa/workunits/ceph-helpers.sh:999: wait_for_clean: expr 91 + 1 ../qa/workunits/ceph-helpers.sh:999: wait_for_clean: timer=92 ../qa/workunits/ceph-helpers.sh:985: wait_for_clean: is_clean .../qa/workunits/ceph-helpers.sh:952: is_clean: get_num_pgs .../qa/workunits/ceph-helpers.sh:898: get_num_pgs: ceph --format xml status .../qa/workunits/ceph-helpers.sh:899: get_num_pgs: xmlstarlet sel -t -m //pgmap/num_pgs -v . ../qa/workunits/ceph-helpers.sh:952: is_clean: num_pgs=4 ../qa/workunits/ceph-helpers.sh:953: is_clean: test 4 '!=' 0 .../qa/workunits/ceph-helpers.sh:954: is_clean: get_num_active_clean .../qa/workunits/ceph-helpers.sh:863: get_num_active_clean: local 'expression=(' .../qa/workunits/ceph-helpers.sh:864: get_num_active_clean: expression+='contains(.,'\''active'\'') and ' .../qa/workunits/ceph-helpers.sh:865: get_num_active_clean: expression+='contains(.,'\''clean'\'') and ' .../qa/workunits/ceph-helpers.sh:866: get_num_active_clean: expression+='not(contains(.,'\''stale'\''))' .../qa/workunits/ceph-helpers.sh:867: get_num_active_clean: expression+=')' .../qa/workunits/ceph-helpers.sh:871: get_num_active_clean: ceph --format xml pg dump pgs .../qa/workunits/ceph-helpers.sh:872: get_num_active_clean: xmlstarlet sel -t -m '//pg_stat/state[(contains(.,'\''active'\'') and contains(.,'\''clean'\'') and not(contains(.,'\''stale'\'')))]' -v . -n .../qa/workunits/ceph-helpers.sh:873: get_num_active_clean: wc -l .../qa/workunits/ceph-helpers.sh:873: get_num_active_clean: grep -v '^$' ../qa/workunits/ceph-helpers.sh:954: is_clean: test 0 = 4 ../qa/workunits/ceph-helpers.sh:954: is_clean: return 1 ../qa/workunits/ceph-helpers.sh:986: wait_for_clean: get_is_making_recovery_progress .../qa/workunits/ceph-helpers.sh:839: get_is_making_recovery_progress: ceph --format xml status .../qa/workunits/ceph-helpers.sh:839: get_is_making_recovery_progress: xmlstarlet sel -t -m //pgmap/recovering_keys_per_sec -v . -o ' ' -t -m //pgmap/recovering_bytes_per_sec -v . -o ' ' -t -m //pgmap/recovering_objects_per_sec -v . ../qa/workunits/ceph-helpers.sh:839: get_is_making_recovery_progress: local progress= ../qa/workunits/ceph-helpers.sh:840: get_is_making_recovery_progress: test -n '' ../qa/workunits/ceph-helpers.sh:988: wait_for_clean: (( timer >= 120 )) .../qa/workunits/ceph-helpers.sh:993: wait_for_clean: get_num_active_clean .../qa/workunits/ceph-helpers.sh:863: get_num_active_clean: local 'expression=(' .../qa/workunits/ceph-helpers.sh:864: get_num_active_clean: expression+='contains(.,'\''active'\'') and ' .../qa/workunits/ceph-helpers.sh:865: get_num_active_clean: expression+='contains(.,'\''clean'\'') and ' .../qa/workunits/ceph-helpers.sh:866: get_num_active_clean: expression+='not(contains(.,'\''stale'\''))' .../qa/workunits/ceph-helpers.sh:867: get_num_active_clean: expression+=')' .../qa/workunits/ceph-helpers.sh:871: get_num_active_clean: ceph --format xml pg dump pgs .../qa/workunits/ceph-helpers.sh:872: get_num_active_clean: xmlstarlet sel -t -m '//pg_stat/state[(contains(.,'\''active'\'') and contains(.,'\''clean'\'') and not(contains(.,'\''stale'\'')))]' -v . -n .../qa/workunits/ceph-helpers.sh:873: get_num_active_clean: grep -v '^$' .../qa/workunits/ceph-helpers.sh:873: get_num_active_clean: wc -l ../qa/workunits/ceph-helpers.sh:993: wait_for_clean: cur_active_clean=0 ../qa/workunits/ceph-helpers.sh:994: wait_for_clean: test 0 '!=' 0 ../qa/workunits/ceph-helpers.sh:998: wait_for_clean: sleep 1 .../qa/workunits/ceph-helpers.sh:999: wait_for_clean: expr 92 + 1 ../qa/workunits/ceph-helpers.sh:999: wait_for_clean: timer=93 ../qa/workunits/ceph-helpers.sh:985: wait_for_clean: is_clean .../qa/workunits/ceph-helpers.sh:952: is_clean: get_num_pgs .../qa/workunits/ceph-helpers.sh:898: get_num_pgs: ceph --format xml status .../qa/workunits/ceph-helpers.sh:899: get_num_pgs: xmlstarlet sel -t -m //pgmap/num_pgs -v . ../qa/workunits/ceph-helpers.sh:952: is_clean: num_pgs=4 ../qa/workunits/ceph-helpers.sh:953: is_clean: test 4 '!=' 0 .../qa/workunits/ceph-helpers.sh:954: is_clean: get_num_active_clean .../qa/workunits/ceph-helpers.sh:863: get_num_active_clean: local 'expression=(' .../qa/workunits/ceph-helpers.sh:864: get_num_active_clean: expression+='contains(.,'\''active'\'') and ' .../qa/workunits/ceph-helpers.sh:865: get_num_active_clean: expression+='contains(.,'\''clean'\'') and ' .../qa/workunits/ceph-helpers.sh:866: get_num_active_clean: expression+='not(contains(.,'\''stale'\''))' .../qa/workunits/ceph-helpers.sh:867: get_num_active_clean: expression+=')' .../qa/workunits/ceph-helpers.sh:871: get_num_active_clean: ceph --format xml pg dump pgs .../qa/workunits/ceph-helpers.sh:872: get_num_active_clean: xmlstarlet sel -t -m '//pg_stat/state[(contains(.,'\''active'\'') and contains(.,'\''clean'\'') and not(contains(.,'\''stale'\'')))]' -v . -n .../qa/workunits/ceph-helpers.sh:873: get_num_active_clean: grep -v '^$' .../qa/workunits/ceph-helpers.sh:873: get_num_active_clean: wc -l ../qa/workunits/ceph-helpers.sh:954: is_clean: test 0 = 4 ../qa/workunits/ceph-helpers.sh:954: is_clean: return 1 ../qa/workunits/ceph-helpers.sh:986: wait_for_clean: get_is_making_recovery_progress .../qa/workunits/ceph-helpers.sh:839: get_is_making_recovery_progress: ceph --format xml status .../qa/workunits/ceph-helpers.sh:839: get_is_making_recovery_progress: xmlstarlet sel -t -m //pgmap/recovering_keys_per_sec -v . -o ' ' -t -m //pgmap/recovering_bytes_per_sec -v . -o ' ' -t -m //pgmap/recovering_objects_per_sec -v . ../qa/workunits/ceph-helpers.sh:839: get_is_making_recovery_progress: local progress= ../qa/workunits/ceph-helpers.sh:840: get_is_making_recovery_progress: test -n '' ../qa/workunits/ceph-helpers.sh:988: wait_for_clean: (( timer >= 120 )) .../qa/workunits/ceph-helpers.sh:993: wait_for_clean: get_num_active_clean .../qa/workunits/ceph-helpers.sh:863: get_num_active_clean: local 'expression=(' .../qa/workunits/ceph-helpers.sh:864: get_num_active_clean: expression+='contains(.,'\''active'\'') and ' .../qa/workunits/ceph-helpers.sh:865: get_num_active_clean: expression+='contains(.,'\''clean'\'') and ' .../qa/workunits/ceph-helpers.sh:866: get_num_active_clean: expression+='not(contains(.,'\''stale'\''))' .../qa/workunits/ceph-helpers.sh:867: get_num_active_clean: expression+=')' .../qa/workunits/ceph-helpers.sh:871: get_num_active_clean: ceph --format xml pg dump pgs .../qa/workunits/ceph-helpers.sh:872: get_num_active_clean: xmlstarlet sel -t -m '//pg_stat/state[(contains(.,'\''active'\'') and contains(.,'\''clean'\'') and not(contains(.,'\''stale'\'')))]' -v . -n .../qa/workunits/ceph-helpers.sh:873: get_num_active_clean: grep -v '^$' .../qa/workunits/ceph-helpers.sh:873: get_num_active_clean: wc -l ../qa/workunits/ceph-helpers.sh:993: wait_for_clean: cur_active_clean=0 ../qa/workunits/ceph-helpers.sh:994: wait_for_clean: test 0 '!=' 0 ../qa/workunits/ceph-helpers.sh:998: wait_for_clean: sleep 1 .../qa/workunits/ceph-helpers.sh:999: wait_for_clean: expr 93 + 1 ../qa/workunits/ceph-helpers.sh:999: wait_for_clean: timer=94 ../qa/workunits/ceph-helpers.sh:985: wait_for_clean: is_clean .../qa/workunits/ceph-helpers.sh:952: is_clean: get_num_pgs .../qa/workunits/ceph-helpers.sh:898: get_num_pgs: ceph --format xml status .../qa/workunits/ceph-helpers.sh:899: get_num_pgs: xmlstarlet sel -t -m //pgmap/num_pgs -v . ../qa/workunits/ceph-helpers.sh:952: is_clean: num_pgs=4 ../qa/workunits/ceph-helpers.sh:953: is_clean: test 4 '!=' 0 .../qa/workunits/ceph-helpers.sh:954: is_clean: get_num_active_clean .../qa/workunits/ceph-helpers.sh:863: get_num_active_clean: local 'expression=(' .../qa/workunits/ceph-helpers.sh:864: get_num_active_clean: expression+='contains(.,'\''active'\'') and ' .../qa/workunits/ceph-helpers.sh:865: get_num_active_clean: expression+='contains(.,'\''clean'\'') and ' .../qa/workunits/ceph-helpers.sh:866: get_num_active_clean: expression+='not(contains(.,'\''stale'\''))' .../qa/workunits/ceph-helpers.sh:867: get_num_active_clean: expression+=')' .../qa/workunits/ceph-helpers.sh:871: get_num_active_clean: ceph --format xml pg dump pgs .../qa/workunits/ceph-helpers.sh:872: get_num_active_clean: xmlstarlet sel -t -m '//pg_stat/state[(contains(.,'\''active'\'') and contains(.,'\''clean'\'') and not(contains(.,'\''stale'\'')))]' -v . -n .../qa/workunits/ceph-helpers.sh:873: get_num_active_clean: wc -l .../qa/workunits/ceph-helpers.sh:873: get_num_active_clean: grep -v '^$' ../qa/workunits/ceph-helpers.sh:954: is_clean: test 0 = 4 ../qa/workunits/ceph-helpers.sh:954: is_clean: return 1 ../qa/workunits/ceph-helpers.sh:986: wait_for_clean: get_is_making_recovery_progress .../qa/workunits/ceph-helpers.sh:839: get_is_making_recovery_progress: ceph --format xml status .../qa/workunits/ceph-helpers.sh:839: get_is_making_recovery_progress: xmlstarlet sel -t -m //pgmap/recovering_keys_per_sec -v . -o ' ' -t -m //pgmap/recovering_bytes_per_sec -v . -o ' ' -t -m //pgmap/recovering_objects_per_sec -v . ../qa/workunits/ceph-helpers.sh:839: get_is_making_recovery_progress: local progress= ../qa/workunits/ceph-helpers.sh:840: get_is_making_recovery_progress: test -n '' ../qa/workunits/ceph-helpers.sh:988: wait_for_clean: (( timer >= 120 )) .../qa/workunits/ceph-helpers.sh:993: wait_for_clean: get_num_active_clean .../qa/workunits/ceph-helpers.sh:863: get_num_active_clean: local 'expression=(' .../qa/workunits/ceph-helpers.sh:864: get_num_active_clean: expression+='contains(.,'\''active'\'') and ' .../qa/workunits/ceph-helpers.sh:865: get_num_active_clean: expression+='contains(.,'\''clean'\'') and ' .../qa/workunits/ceph-helpers.sh:866: get_num_active_clean: expression+='not(contains(.,'\''stale'\''))' .../qa/workunits/ceph-helpers.sh:867: get_num_active_clean: expression+=')' .../qa/workunits/ceph-helpers.sh:871: get_num_active_clean: ceph --format xml pg dump pgs .../qa/workunits/ceph-helpers.sh:872: get_num_active_clean: xmlstarlet sel -t -m '//pg_stat/state[(contains(.,'\''active'\'') and contains(.,'\''clean'\'') and not(contains(.,'\''stale'\'')))]' -v . -n .../qa/workunits/ceph-helpers.sh:873: get_num_active_clean: wc -l .../qa/workunits/ceph-helpers.sh:873: get_num_active_clean: grep -v '^$' ../qa/workunits/ceph-helpers.sh:993: wait_for_clean: cur_active_clean=0 ../qa/workunits/ceph-helpers.sh:994: wait_for_clean: test 0 '!=' 0 ../qa/workunits/ceph-helpers.sh:998: wait_for_clean: sleep 1 .../qa/workunits/ceph-helpers.sh:999: wait_for_clean: expr 94 + 1 ../qa/workunits/ceph-helpers.sh:999: wait_for_clean: timer=95 ../qa/workunits/ceph-helpers.sh:985: wait_for_clean: is_clean .../qa/workunits/ceph-helpers.sh:952: is_clean: get_num_pgs .../qa/workunits/ceph-helpers.sh:898: get_num_pgs: ceph --format xml status .../qa/workunits/ceph-helpers.sh:899: get_num_pgs: xmlstarlet sel -t -m //pgmap/num_pgs -v . ../qa/workunits/ceph-helpers.sh:952: is_clean: num_pgs=4 ../qa/workunits/ceph-helpers.sh:953: is_clean: test 4 '!=' 0 .../qa/workunits/ceph-helpers.sh:954: is_clean: get_num_active_clean .../qa/workunits/ceph-helpers.sh:863: get_num_active_clean: local 'expression=(' .../qa/workunits/ceph-helpers.sh:864: get_num_active_clean: expression+='contains(.,'\''active'\'') and ' .../qa/workunits/ceph-helpers.sh:865: get_num_active_clean: expression+='contains(.,'\''clean'\'') and ' .../qa/workunits/ceph-helpers.sh:866: get_num_active_clean: expression+='not(contains(.,'\''stale'\''))' .../qa/workunits/ceph-helpers.sh:867: get_num_active_clean: expression+=')' .../qa/workunits/ceph-helpers.sh:871: get_num_active_clean: ceph --format xml pg dump pgs .../qa/workunits/ceph-helpers.sh:872: get_num_active_clean: xmlstarlet sel -t -m '//pg_stat/state[(contains(.,'\''active'\'') and contains(.,'\''clean'\'') and not(contains(.,'\''stale'\'')))]' -v . -n .../qa/workunits/ceph-helpers.sh:873: get_num_active_clean: grep -v '^$' .../qa/workunits/ceph-helpers.sh:873: get_num_active_clean: wc -l ../qa/workunits/ceph-helpers.sh:954: is_clean: test 0 = 4 ../qa/workunits/ceph-helpers.sh:954: is_clean: return 1 ../qa/workunits/ceph-helpers.sh:986: wait_for_clean: get_is_making_recovery_progress .../qa/workunits/ceph-helpers.sh:839: get_is_making_recovery_progress: ceph --format xml status .../qa/workunits/ceph-helpers.sh:839: get_is_making_recovery_progress: xmlstarlet sel -t -m //pgmap/recovering_keys_per_sec -v . -o ' ' -t -m //pgmap/recovering_bytes_per_sec -v . -o ' ' -t -m //pgmap/recovering_objects_per_sec -v . ../qa/workunits/ceph-helpers.sh:839: get_is_making_recovery_progress: local progress= ../qa/workunits/ceph-helpers.sh:840: get_is_making_recovery_progress: test -n '' ../qa/workunits/ceph-helpers.sh:988: wait_for_clean: (( timer >= 120 )) .../qa/workunits/ceph-helpers.sh:993: wait_for_clean: get_num_active_clean .../qa/workunits/ceph-helpers.sh:863: get_num_active_clean: local 'expression=(' .../qa/workunits/ceph-helpers.sh:864: get_num_active_clean: expression+='contains(.,'\''active'\'') and ' .../qa/workunits/ceph-helpers.sh:865: get_num_active_clean: expression+='contains(.,'\''clean'\'') and ' .../qa/workunits/ceph-helpers.sh:866: get_num_active_clean: expression+='not(contains(.,'\''stale'\''))' .../qa/workunits/ceph-helpers.sh:867: get_num_active_clean: expression+=')' .../qa/workunits/ceph-helpers.sh:871: get_num_active_clean: ceph --format xml pg dump pgs .../qa/workunits/ceph-helpers.sh:872: get_num_active_clean: xmlstarlet sel -t -m '//pg_stat/state[(contains(.,'\''active'\'') and contains(.,'\''clean'\'') and not(contains(.,'\''stale'\'')))]' -v . -n .../qa/workunits/ceph-helpers.sh:873: get_num_active_clean: grep -v '^$' .../qa/workunits/ceph-helpers.sh:873: get_num_active_clean: wc -l ../qa/workunits/ceph-helpers.sh:993: wait_for_clean: cur_active_clean=0 ../qa/workunits/ceph-helpers.sh:994: wait_for_clean: test 0 '!=' 0 ../qa/workunits/ceph-helpers.sh:998: wait_for_clean: sleep 1 .../qa/workunits/ceph-helpers.sh:999: wait_for_clean: expr 95 + 1 ../qa/workunits/ceph-helpers.sh:999: wait_for_clean: timer=96 ../qa/workunits/ceph-helpers.sh:985: wait_for_clean: is_clean .../qa/workunits/ceph-helpers.sh:952: is_clean: get_num_pgs .../qa/workunits/ceph-helpers.sh:898: get_num_pgs: ceph --format xml status .../qa/workunits/ceph-helpers.sh:899: get_num_pgs: xmlstarlet sel -t -m //pgmap/num_pgs -v . ../qa/workunits/ceph-helpers.sh:952: is_clean: num_pgs=4 ../qa/workunits/ceph-helpers.sh:953: is_clean: test 4 '!=' 0 .../qa/workunits/ceph-helpers.sh:954: is_clean: get_num_active_clean .../qa/workunits/ceph-helpers.sh:863: get_num_active_clean: local 'expression=(' .../qa/workunits/ceph-helpers.sh:864: get_num_active_clean: expression+='contains(.,'\''active'\'') and ' .../qa/workunits/ceph-helpers.sh:865: get_num_active_clean: expression+='contains(.,'\''clean'\'') and ' .../qa/workunits/ceph-helpers.sh:866: get_num_active_clean: expression+='not(contains(.,'\''stale'\''))' .../qa/workunits/ceph-helpers.sh:867: get_num_active_clean: expression+=')' .../qa/workunits/ceph-helpers.sh:871: get_num_active_clean: ceph --format xml pg dump pgs .../qa/workunits/ceph-helpers.sh:872: get_num_active_clean: xmlstarlet sel -t -m '//pg_stat/state[(contains(.,'\''active'\'') and contains(.,'\''clean'\'') and not(contains(.,'\''stale'\'')))]' -v . -n .../qa/workunits/ceph-helpers.sh:873: get_num_active_clean: grep -v '^$' .../qa/workunits/ceph-helpers.sh:873: get_num_active_clean: wc -l ../qa/workunits/ceph-helpers.sh:954: is_clean: test 0 = 4 ../qa/workunits/ceph-helpers.sh:954: is_clean: return 1 ../qa/workunits/ceph-helpers.sh:986: wait_for_clean: get_is_making_recovery_progress .../qa/workunits/ceph-helpers.sh:839: get_is_making_recovery_progress: ceph --format xml status .../qa/workunits/ceph-helpers.sh:839: get_is_making_recovery_progress: xmlstarlet sel -t -m //pgmap/recovering_keys_per_sec -v . -o ' ' -t -m //pgmap/recovering_bytes_per_sec -v . -o ' ' -t -m //pgmap/recovering_objects_per_sec -v . ../qa/workunits/ceph-helpers.sh:839: get_is_making_recovery_progress: local progress= ../qa/workunits/ceph-helpers.sh:840: get_is_making_recovery_progress: test -n '' ../qa/workunits/ceph-helpers.sh:988: wait_for_clean: (( timer >= 120 )) .../qa/workunits/ceph-helpers.sh:993: wait_for_clean: get_num_active_clean .../qa/workunits/ceph-helpers.sh:863: get_num_active_clean: local 'expression=(' .../qa/workunits/ceph-helpers.sh:864: get_num_active_clean: expression+='contains(.,'\''active'\'') and ' .../qa/workunits/ceph-helpers.sh:865: get_num_active_clean: expression+='contains(.,'\''clean'\'') and ' .../qa/workunits/ceph-helpers.sh:866: get_num_active_clean: expression+='not(contains(.,'\''stale'\''))' .../qa/workunits/ceph-helpers.sh:867: get_num_active_clean: expression+=')' .../qa/workunits/ceph-helpers.sh:871: get_num_active_clean: ceph --format xml pg dump pgs .../qa/workunits/ceph-helpers.sh:872: get_num_active_clean: xmlstarlet sel -t -m '//pg_stat/state[(contains(.,'\''active'\'') and contains(.,'\''clean'\'') and not(contains(.,'\''stale'\'')))]' -v . -n .../qa/workunits/ceph-helpers.sh:873: get_num_active_clean: grep -v '^$' .../qa/workunits/ceph-helpers.sh:873: get_num_active_clean: wc -l ../qa/workunits/ceph-helpers.sh:993: wait_for_clean: cur_active_clean=0 ../qa/workunits/ceph-helpers.sh:994: wait_for_clean: test 0 '!=' 0 ../qa/workunits/ceph-helpers.sh:998: wait_for_clean: sleep 1 .../qa/workunits/ceph-helpers.sh:999: wait_for_clean: expr 96 + 1 ../qa/workunits/ceph-helpers.sh:999: wait_for_clean: timer=97 ../qa/workunits/ceph-helpers.sh:985: wait_for_clean: is_clean .../qa/workunits/ceph-helpers.sh:952: is_clean: get_num_pgs .../qa/workunits/ceph-helpers.sh:898: get_num_pgs: ceph --format xml status .../qa/workunits/ceph-helpers.sh:899: get_num_pgs: xmlstarlet sel -t -m //pgmap/num_pgs -v . ../qa/workunits/ceph-helpers.sh:952: is_clean: num_pgs=4 ../qa/workunits/ceph-helpers.sh:953: is_clean: test 4 '!=' 0 .../qa/workunits/ceph-helpers.sh:954: is_clean: get_num_active_clean .../qa/workunits/ceph-helpers.sh:863: get_num_active_clean: local 'expression=(' .../qa/workunits/ceph-helpers.sh:864: get_num_active_clean: expression+='contains(.,'\''active'\'') and ' .../qa/workunits/ceph-helpers.sh:865: get_num_active_clean: expression+='contains(.,'\''clean'\'') and ' .../qa/workunits/ceph-helpers.sh:866: get_num_active_clean: expression+='not(contains(.,'\''stale'\''))' .../qa/workunits/ceph-helpers.sh:867: get_num_active_clean: expression+=')' .../qa/workunits/ceph-helpers.sh:871: get_num_active_clean: ceph --format xml pg dump pgs .../qa/workunits/ceph-helpers.sh:872: get_num_active_clean: xmlstarlet sel -t -m '//pg_stat/state[(contains(.,'\''active'\'') and contains(.,'\''clean'\'') and not(contains(.,'\''stale'\'')))]' -v . -n .../qa/workunits/ceph-helpers.sh:873: get_num_active_clean: wc -l .../qa/workunits/ceph-helpers.sh:873: get_num_active_clean: grep -v '^$' ../qa/workunits/ceph-helpers.sh:954: is_clean: test 0 = 4 ../qa/workunits/ceph-helpers.sh:954: is_clean: return 1 ../qa/workunits/ceph-helpers.sh:986: wait_for_clean: get_is_making_recovery_progress .../qa/workunits/ceph-helpers.sh:839: get_is_making_recovery_progress: ceph --format xml status .../qa/workunits/ceph-helpers.sh:839: get_is_making_recovery_progress: xmlstarlet sel -t -m //pgmap/recovering_keys_per_sec -v . -o ' ' -t -m //pgmap/recovering_bytes_per_sec -v . -o ' ' -t -m //pgmap/recovering_objects_per_sec -v . ../qa/workunits/ceph-helpers.sh:839: get_is_making_recovery_progress: local progress= ../qa/workunits/ceph-helpers.sh:840: get_is_making_recovery_progress: test -n '' ../qa/workunits/ceph-helpers.sh:988: wait_for_clean: (( timer >= 120 )) .../qa/workunits/ceph-helpers.sh:993: wait_for_clean: get_num_active_clean .../qa/workunits/ceph-helpers.sh:863: get_num_active_clean: local 'expression=(' .../qa/workunits/ceph-helpers.sh:864: get_num_active_clean: expression+='contains(.,'\''active'\'') and ' .../qa/workunits/ceph-helpers.sh:865: get_num_active_clean: expression+='contains(.,'\''clean'\'') and ' .../qa/workunits/ceph-helpers.sh:866: get_num_active_clean: expression+='not(contains(.,'\''stale'\''))' .../qa/workunits/ceph-helpers.sh:867: get_num_active_clean: expression+=')' .../qa/workunits/ceph-helpers.sh:871: get_num_active_clean: ceph --format xml pg dump pgs .../qa/workunits/ceph-helpers.sh:872: get_num_active_clean: xmlstarlet sel -t -m '//pg_stat/state[(contains(.,'\''active'\'') and contains(.,'\''clean'\'') and not(contains(.,'\''stale'\'')))]' -v . -n .../qa/workunits/ceph-helpers.sh:873: get_num_active_clean: grep -v '^$' .../qa/workunits/ceph-helpers.sh:873: get_num_active_clean: wc -l ../qa/workunits/ceph-helpers.sh:993: wait_for_clean: cur_active_clean=0 ../qa/workunits/ceph-helpers.sh:994: wait_for_clean: test 0 '!=' 0 ../qa/workunits/ceph-helpers.sh:998: wait_for_clean: sleep 1 .../qa/workunits/ceph-helpers.sh:999: wait_for_clean: expr 97 + 1 ../qa/workunits/ceph-helpers.sh:999: wait_for_clean: timer=98 ../qa/workunits/ceph-helpers.sh:985: wait_for_clean: is_clean .../qa/workunits/ceph-helpers.sh:952: is_clean: get_num_pgs .../qa/workunits/ceph-helpers.sh:898: get_num_pgs: ceph --format xml status .../qa/workunits/ceph-helpers.sh:899: get_num_pgs: xmlstarlet sel -t -m //pgmap/num_pgs -v . ../qa/workunits/ceph-helpers.sh:952: is_clean: num_pgs=4 ../qa/workunits/ceph-helpers.sh:953: is_clean: test 4 '!=' 0 .../qa/workunits/ceph-helpers.sh:954: is_clean: get_num_active_clean .../qa/workunits/ceph-helpers.sh:863: get_num_active_clean: local 'expression=(' .../qa/workunits/ceph-helpers.sh:864: get_num_active_clean: expression+='contains(.,'\''active'\'') and ' .../qa/workunits/ceph-helpers.sh:865: get_num_active_clean: expression+='contains(.,'\''clean'\'') and ' .../qa/workunits/ceph-helpers.sh:866: get_num_active_clean: expression+='not(contains(.,'\''stale'\''))' .../qa/workunits/ceph-helpers.sh:867: get_num_active_clean: expression+=')' .../qa/workunits/ceph-helpers.sh:871: get_num_active_clean: ceph --format xml pg dump pgs .../qa/workunits/ceph-helpers.sh:872: get_num_active_clean: xmlstarlet sel -t -m '//pg_stat/state[(contains(.,'\''active'\'') and contains(.,'\''clean'\'') and not(contains(.,'\''stale'\'')))]' -v . -n .../qa/workunits/ceph-helpers.sh:873: get_num_active_clean: grep -v '^$' .../qa/workunits/ceph-helpers.sh:873: get_num_active_clean: wc -l ../qa/workunits/ceph-helpers.sh:954: is_clean: test 0 = 4 ../qa/workunits/ceph-helpers.sh:954: is_clean: return 1 ../qa/workunits/ceph-helpers.sh:986: wait_for_clean: get_is_making_recovery_progress .../qa/workunits/ceph-helpers.sh:839: get_is_making_recovery_progress: ceph --format xml status .../qa/workunits/ceph-helpers.sh:839: get_is_making_recovery_progress: xmlstarlet sel -t -m //pgmap/recovering_keys_per_sec -v . -o ' ' -t -m //pgmap/recovering_bytes_per_sec -v . -o ' ' -t -m //pgmap/recovering_objects_per_sec -v . ../qa/workunits/ceph-helpers.sh:839: get_is_making_recovery_progress: local progress= ../qa/workunits/ceph-helpers.sh:840: get_is_making_recovery_progress: test -n '' ../qa/workunits/ceph-helpers.sh:988: wait_for_clean: (( timer >= 120 )) .../qa/workunits/ceph-helpers.sh:993: wait_for_clean: get_num_active_clean .../qa/workunits/ceph-helpers.sh:863: get_num_active_clean: local 'expression=(' .../qa/workunits/ceph-helpers.sh:864: get_num_active_clean: expression+='contains(.,'\''active'\'') and ' .../qa/workunits/ceph-helpers.sh:865: get_num_active_clean: expression+='contains(.,'\''clean'\'') and ' .../qa/workunits/ceph-helpers.sh:866: get_num_active_clean: expression+='not(contains(.,'\''stale'\''))' .../qa/workunits/ceph-helpers.sh:867: get_num_active_clean: expression+=')' .../qa/workunits/ceph-helpers.sh:871: get_num_active_clean: ceph --format xml pg dump pgs .../qa/workunits/ceph-helpers.sh:872: get_num_active_clean: xmlstarlet sel -t -m '//pg_stat/state[(contains(.,'\''active'\'') and contains(.,'\''clean'\'') and not(contains(.,'\''stale'\'')))]' -v . -n .../qa/workunits/ceph-helpers.sh:873: get_num_active_clean: grep -v '^$' .../qa/workunits/ceph-helpers.sh:873: get_num_active_clean: wc -l ../qa/workunits/ceph-helpers.sh:993: wait_for_clean: cur_active_clean=0 ../qa/workunits/ceph-helpers.sh:994: wait_for_clean: test 0 '!=' 0 ../qa/workunits/ceph-helpers.sh:998: wait_for_clean: sleep 1 .../qa/workunits/ceph-helpers.sh:999: wait_for_clean: expr 98 + 1 ../qa/workunits/ceph-helpers.sh:999: wait_for_clean: timer=99 ../qa/workunits/ceph-helpers.sh:985: wait_for_clean: is_clean .../qa/workunits/ceph-helpers.sh:952: is_clean: get_num_pgs .../qa/workunits/ceph-helpers.sh:898: get_num_pgs: ceph --format xml status .../qa/workunits/ceph-helpers.sh:899: get_num_pgs: xmlstarlet sel -t -m //pgmap/num_pgs -v . ../qa/workunits/ceph-helpers.sh:952: is_clean: num_pgs=4 ../qa/workunits/ceph-helpers.sh:953: is_clean: test 4 '!=' 0 .../qa/workunits/ceph-helpers.sh:954: is_clean: get_num_active_clean .../qa/workunits/ceph-helpers.sh:863: get_num_active_clean: local 'expression=(' .../qa/workunits/ceph-helpers.sh:864: get_num_active_clean: expression+='contains(.,'\''active'\'') and ' .../qa/workunits/ceph-helpers.sh:865: get_num_active_clean: expression+='contains(.,'\''clean'\'') and ' .../qa/workunits/ceph-helpers.sh:866: get_num_active_clean: expression+='not(contains(.,'\''stale'\''))' .../qa/workunits/ceph-helpers.sh:867: get_num_active_clean: expression+=')' .../qa/workunits/ceph-helpers.sh:871: get_num_active_clean: ceph --format xml pg dump pgs .../qa/workunits/ceph-helpers.sh:872: get_num_active_clean: xmlstarlet sel -t -m '//pg_stat/state[(contains(.,'\''active'\'') and contains(.,'\''clean'\'') and not(contains(.,'\''stale'\'')))]' -v . -n .../qa/workunits/ceph-helpers.sh:873: get_num_active_clean: grep -v '^$' .../qa/workunits/ceph-helpers.sh:873: get_num_active_clean: wc -l ../qa/workunits/ceph-helpers.sh:954: is_clean: test 0 = 4 ../qa/workunits/ceph-helpers.sh:954: is_clean: return 1 ../qa/workunits/ceph-helpers.sh:986: wait_for_clean: get_is_making_recovery_progress .../qa/workunits/ceph-helpers.sh:839: get_is_making_recovery_progress: ceph --format xml status .../qa/workunits/ceph-helpers.sh:839: get_is_making_recovery_progress: xmlstarlet sel -t -m //pgmap/recovering_keys_per_sec -v . -o ' ' -t -m //pgmap/recovering_bytes_per_sec -v . -o ' ' -t -m //pgmap/recovering_objects_per_sec -v . ../qa/workunits/ceph-helpers.sh:839: get_is_making_recovery_progress: local progress= ../qa/workunits/ceph-helpers.sh:840: get_is_making_recovery_progress: test -n '' ../qa/workunits/ceph-helpers.sh:988: wait_for_clean: (( timer >= 120 )) .../qa/workunits/ceph-helpers.sh:993: wait_for_clean: get_num_active_clean .../qa/workunits/ceph-helpers.sh:863: get_num_active_clean: local 'expression=(' .../qa/workunits/ceph-helpers.sh:864: get_num_active_clean: expression+='contains(.,'\''active'\'') and ' .../qa/workunits/ceph-helpers.sh:865: get_num_active_clean: expression+='contains(.,'\''clean'\'') and ' .../qa/workunits/ceph-helpers.sh:866: get_num_active_clean: expression+='not(contains(.,'\''stale'\''))' .../qa/workunits/ceph-helpers.sh:867: get_num_active_clean: expression+=')' .../qa/workunits/ceph-helpers.sh:871: get_num_active_clean: ceph --format xml pg dump pgs .../qa/workunits/ceph-helpers.sh:872: get_num_active_clean: xmlstarlet sel -t -m '//pg_stat/state[(contains(.,'\''active'\'') and contains(.,'\''clean'\'') and not(contains(.,'\''stale'\'')))]' -v . -n .../qa/workunits/ceph-helpers.sh:873: get_num_active_clean: grep -v '^$' .../qa/workunits/ceph-helpers.sh:873: get_num_active_clean: wc -l ../qa/workunits/ceph-helpers.sh:993: wait_for_clean: cur_active_clean=0 ../qa/workunits/ceph-helpers.sh:994: wait_for_clean: test 0 '!=' 0 ../qa/workunits/ceph-helpers.sh:998: wait_for_clean: sleep 1 .../qa/workunits/ceph-helpers.sh:999: wait_for_clean: expr 99 + 1 ../qa/workunits/ceph-helpers.sh:999: wait_for_clean: timer=100 ../qa/workunits/ceph-helpers.sh:985: wait_for_clean: is_clean .../qa/workunits/ceph-helpers.sh:952: is_clean: get_num_pgs .../qa/workunits/ceph-helpers.sh:898: get_num_pgs: ceph --format xml status .../qa/workunits/ceph-helpers.sh:899: get_num_pgs: xmlstarlet sel -t -m //pgmap/num_pgs -v . ../qa/workunits/ceph-helpers.sh:952: is_clean: num_pgs=4 ../qa/workunits/ceph-helpers.sh:953: is_clean: test 4 '!=' 0 .../qa/workunits/ceph-helpers.sh:954: is_clean: get_num_active_clean .../qa/workunits/ceph-helpers.sh:863: get_num_active_clean: local 'expression=(' .../qa/workunits/ceph-helpers.sh:864: get_num_active_clean: expression+='contains(.,'\''active'\'') and ' .../qa/workunits/ceph-helpers.sh:865: get_num_active_clean: expression+='contains(.,'\''clean'\'') and ' .../qa/workunits/ceph-helpers.sh:866: get_num_active_clean: expression+='not(contains(.,'\''stale'\''))' .../qa/workunits/ceph-helpers.sh:867: get_num_active_clean: expression+=')' .../qa/workunits/ceph-helpers.sh:871: get_num_active_clean: ceph --format xml pg dump pgs .../qa/workunits/ceph-helpers.sh:872: get_num_active_clean: xmlstarlet sel -t -m '//pg_stat/state[(contains(.,'\''active'\'') and contains(.,'\''clean'\'') and not(contains(.,'\''stale'\'')))]' -v . -n .../qa/workunits/ceph-helpers.sh:873: get_num_active_clean: wc -l .../qa/workunits/ceph-helpers.sh:873: get_num_active_clean: grep -v '^$' ../qa/workunits/ceph-helpers.sh:954: is_clean: test 0 = 4 ../qa/workunits/ceph-helpers.sh:954: is_clean: return 1 ../qa/workunits/ceph-helpers.sh:986: wait_for_clean: get_is_making_recovery_progress .../qa/workunits/ceph-helpers.sh:839: get_is_making_recovery_progress: ceph --format xml status .../qa/workunits/ceph-helpers.sh:839: get_is_making_recovery_progress: xmlstarlet sel -t -m //pgmap/recovering_keys_per_sec -v . -o ' ' -t -m //pgmap/recovering_bytes_per_sec -v . -o ' ' -t -m //pgmap/recovering_objects_per_sec -v . ../qa/workunits/ceph-helpers.sh:839: get_is_making_recovery_progress: local progress= ../qa/workunits/ceph-helpers.sh:840: get_is_making_recovery_progress: test -n '' ../qa/workunits/ceph-helpers.sh:988: wait_for_clean: (( timer >= 120 )) .../qa/workunits/ceph-helpers.sh:993: wait_for_clean: get_num_active_clean .../qa/workunits/ceph-helpers.sh:863: get_num_active_clean: local 'expression=(' .../qa/workunits/ceph-helpers.sh:864: get_num_active_clean: expression+='contains(.,'\''active'\'') and ' .../qa/workunits/ceph-helpers.sh:865: get_num_active_clean: expression+='contains(.,'\''clean'\'') and ' .../qa/workunits/ceph-helpers.sh:866: get_num_active_clean: expression+='not(contains(.,'\''stale'\''))' .../qa/workunits/ceph-helpers.sh:867: get_num_active_clean: expression+=')' .../qa/workunits/ceph-helpers.sh:871: get_num_active_clean: ceph --format xml pg dump pgs .../qa/workunits/ceph-helpers.sh:872: get_num_active_clean: xmlstarlet sel -t -m '//pg_stat/state[(contains(.,'\''active'\'') and contains(.,'\''clean'\'') and not(contains(.,'\''stale'\'')))]' -v . -n .../qa/workunits/ceph-helpers.sh:873: get_num_active_clean: wc -l .../qa/workunits/ceph-helpers.sh:873: get_num_active_clean: grep -v '^$' ../qa/workunits/ceph-helpers.sh:993: wait_for_clean: cur_active_clean=0 ../qa/workunits/ceph-helpers.sh:994: wait_for_clean: test 0 '!=' 0 ../qa/workunits/ceph-helpers.sh:998: wait_for_clean: sleep 1 .../qa/workunits/ceph-helpers.sh:999: wait_for_clean: expr 100 + 1 ../qa/workunits/ceph-helpers.sh:999: wait_for_clean: timer=101 ../qa/workunits/ceph-helpers.sh:985: wait_for_clean: is_clean .../qa/workunits/ceph-helpers.sh:952: is_clean: get_num_pgs .../qa/workunits/ceph-helpers.sh:898: get_num_pgs: ceph --format xml status .../qa/workunits/ceph-helpers.sh:899: get_num_pgs: xmlstarlet sel -t -m //pgmap/num_pgs -v . ../qa/workunits/ceph-helpers.sh:952: is_clean: num_pgs=4 ../qa/workunits/ceph-helpers.sh:953: is_clean: test 4 '!=' 0 .../qa/workunits/ceph-helpers.sh:954: is_clean: get_num_active_clean .../qa/workunits/ceph-helpers.sh:863: get_num_active_clean: local 'expression=(' .../qa/workunits/ceph-helpers.sh:864: get_num_active_clean: expression+='contains(.,'\''active'\'') and ' .../qa/workunits/ceph-helpers.sh:865: get_num_active_clean: expression+='contains(.,'\''clean'\'') and ' .../qa/workunits/ceph-helpers.sh:866: get_num_active_clean: expression+='not(contains(.,'\''stale'\''))' .../qa/workunits/ceph-helpers.sh:867: get_num_active_clean: expression+=')' .../qa/workunits/ceph-helpers.sh:871: get_num_active_clean: ceph --format xml pg dump pgs .../qa/workunits/ceph-helpers.sh:872: get_num_active_clean: xmlstarlet sel -t -m '//pg_stat/state[(contains(.,'\''active'\'') and contains(.,'\''clean'\'') and not(contains(.,'\''stale'\'')))]' -v . -n .../qa/workunits/ceph-helpers.sh:873: get_num_active_clean: grep -v '^$' .../qa/workunits/ceph-helpers.sh:873: get_num_active_clean: wc -l ../qa/workunits/ceph-helpers.sh:954: is_clean: test 0 = 4 ../qa/workunits/ceph-helpers.sh:954: is_clean: return 1 ../qa/workunits/ceph-helpers.sh:986: wait_for_clean: get_is_making_recovery_progress .../qa/workunits/ceph-helpers.sh:839: get_is_making_recovery_progress: ceph --format xml status .../qa/workunits/ceph-helpers.sh:839: get_is_making_recovery_progress: xmlstarlet sel -t -m //pgmap/recovering_keys_per_sec -v . -o ' ' -t -m //pgmap/recovering_bytes_per_sec -v . -o ' ' -t -m //pgmap/recovering_objects_per_sec -v . ../qa/workunits/ceph-helpers.sh:839: get_is_making_recovery_progress: local progress= ../qa/workunits/ceph-helpers.sh:840: get_is_making_recovery_progress: test -n '' ../qa/workunits/ceph-helpers.sh:988: wait_for_clean: (( timer >= 120 )) .../qa/workunits/ceph-helpers.sh:993: wait_for_clean: get_num_active_clean .../qa/workunits/ceph-helpers.sh:863: get_num_active_clean: local 'expression=(' .../qa/workunits/ceph-helpers.sh:864: get_num_active_clean: expression+='contains(.,'\''active'\'') and ' .../qa/workunits/ceph-helpers.sh:865: get_num_active_clean: expression+='contains(.,'\''clean'\'') and ' .../qa/workunits/ceph-helpers.sh:866: get_num_active_clean: expression+='not(contains(.,'\''stale'\''))' .../qa/workunits/ceph-helpers.sh:867: get_num_active_clean: expression+=')' .../qa/workunits/ceph-helpers.sh:871: get_num_active_clean: ceph --format xml pg dump pgs .../qa/workunits/ceph-helpers.sh:872: get_num_active_clean: xmlstarlet sel -t -m '//pg_stat/state[(contains(.,'\''active'\'') and contains(.,'\''clean'\'') and not(contains(.,'\''stale'\'')))]' -v . -n .../qa/workunits/ceph-helpers.sh:873: get_num_active_clean: grep -v '^$' .../qa/workunits/ceph-helpers.sh:873: get_num_active_clean: wc -l ../qa/workunits/ceph-helpers.sh:993: wait_for_clean: cur_active_clean=0 ../qa/workunits/ceph-helpers.sh:994: wait_for_clean: test 0 '!=' 0 ../qa/workunits/ceph-helpers.sh:998: wait_for_clean: sleep 1 .../qa/workunits/ceph-helpers.sh:999: wait_for_clean: expr 101 + 1 ../qa/workunits/ceph-helpers.sh:999: wait_for_clean: timer=102 ../qa/workunits/ceph-helpers.sh:985: wait_for_clean: is_clean .../qa/workunits/ceph-helpers.sh:952: is_clean: get_num_pgs .../qa/workunits/ceph-helpers.sh:898: get_num_pgs: ceph --format xml status .../qa/workunits/ceph-helpers.sh:899: get_num_pgs: xmlstarlet sel -t -m //pgmap/num_pgs -v . ../qa/workunits/ceph-helpers.sh:952: is_clean: num_pgs=4 ../qa/workunits/ceph-helpers.sh:953: is_clean: test 4 '!=' 0 .../qa/workunits/ceph-helpers.sh:954: is_clean: get_num_active_clean .../qa/workunits/ceph-helpers.sh:863: get_num_active_clean: local 'expression=(' .../qa/workunits/ceph-helpers.sh:864: get_num_active_clean: expression+='contains(.,'\''active'\'') and ' .../qa/workunits/ceph-helpers.sh:865: get_num_active_clean: expression+='contains(.,'\''clean'\'') and ' .../qa/workunits/ceph-helpers.sh:866: get_num_active_clean: expression+='not(contains(.,'\''stale'\''))' .../qa/workunits/ceph-helpers.sh:867: get_num_active_clean: expression+=')' .../qa/workunits/ceph-helpers.sh:871: get_num_active_clean: ceph --format xml pg dump pgs .../qa/workunits/ceph-helpers.sh:872: get_num_active_clean: xmlstarlet sel -t -m '//pg_stat/state[(contains(.,'\''active'\'') and contains(.,'\''clean'\'') and not(contains(.,'\''stale'\'')))]' -v . -n .../qa/workunits/ceph-helpers.sh:873: get_num_active_clean: grep -v '^$' .../qa/workunits/ceph-helpers.sh:873: get_num_active_clean: wc -l ../qa/workunits/ceph-helpers.sh:954: is_clean: test 0 = 4 ../qa/workunits/ceph-helpers.sh:954: is_clean: return 1 ../qa/workunits/ceph-helpers.sh:986: wait_for_clean: get_is_making_recovery_progress .../qa/workunits/ceph-helpers.sh:839: get_is_making_recovery_progress: ceph --format xml status .../qa/workunits/ceph-helpers.sh:839: get_is_making_recovery_progress: xmlstarlet sel -t -m //pgmap/recovering_keys_per_sec -v . -o ' ' -t -m //pgmap/recovering_bytes_per_sec -v . -o ' ' -t -m //pgmap/recovering_objects_per_sec -v . ../qa/workunits/ceph-helpers.sh:839: get_is_making_recovery_progress: local progress= ../qa/workunits/ceph-helpers.sh:840: get_is_making_recovery_progress: test -n '' ../qa/workunits/ceph-helpers.sh:988: wait_for_clean: (( timer >= 120 )) .../qa/workunits/ceph-helpers.sh:993: wait_for_clean: get_num_active_clean .../qa/workunits/ceph-helpers.sh:863: get_num_active_clean: local 'expression=(' .../qa/workunits/ceph-helpers.sh:864: get_num_active_clean: expression+='contains(.,'\''active'\'') and ' .../qa/workunits/ceph-helpers.sh:865: get_num_active_clean: expression+='contains(.,'\''clean'\'') and ' .../qa/workunits/ceph-helpers.sh:866: get_num_active_clean: expression+='not(contains(.,'\''stale'\''))' .../qa/workunits/ceph-helpers.sh:867: get_num_active_clean: expression+=')' .../qa/workunits/ceph-helpers.sh:871: get_num_active_clean: ceph --format xml pg dump pgs .../qa/workunits/ceph-helpers.sh:872: get_num_active_clean: xmlstarlet sel -t -m '//pg_stat/state[(contains(.,'\''active'\'') and contains(.,'\''clean'\'') and not(contains(.,'\''stale'\'')))]' -v . -n .../qa/workunits/ceph-helpers.sh:873: get_num_active_clean: grep -v '^$' .../qa/workunits/ceph-helpers.sh:873: get_num_active_clean: wc -l ../qa/workunits/ceph-helpers.sh:993: wait_for_clean: cur_active_clean=0 ../qa/workunits/ceph-helpers.sh:994: wait_for_clean: test 0 '!=' 0 ../qa/workunits/ceph-helpers.sh:998: wait_for_clean: sleep 1 .../qa/workunits/ceph-helpers.sh:999: wait_for_clean: expr 102 + 1 ../qa/workunits/ceph-helpers.sh:999: wait_for_clean: timer=103 ../qa/workunits/ceph-helpers.sh:985: wait_for_clean: is_clean .../qa/workunits/ceph-helpers.sh:952: is_clean: get_num_pgs .../qa/workunits/ceph-helpers.sh:898: get_num_pgs: ceph --format xml status .../qa/workunits/ceph-helpers.sh:899: get_num_pgs: xmlstarlet sel -t -m //pgmap/num_pgs -v . ../qa/workunits/ceph-helpers.sh:952: is_clean: num_pgs=4 ../qa/workunits/ceph-helpers.sh:953: is_clean: test 4 '!=' 0 .../qa/workunits/ceph-helpers.sh:954: is_clean: get_num_active_clean .../qa/workunits/ceph-helpers.sh:863: get_num_active_clean: local 'expression=(' .../qa/workunits/ceph-helpers.sh:864: get_num_active_clean: expression+='contains(.,'\''active'\'') and ' .../qa/workunits/ceph-helpers.sh:865: get_num_active_clean: expression+='contains(.,'\''clean'\'') and ' .../qa/workunits/ceph-helpers.sh:866: get_num_active_clean: expression+='not(contains(.,'\''stale'\''))' .../qa/workunits/ceph-helpers.sh:867: get_num_active_clean: expression+=')' .../qa/workunits/ceph-helpers.sh:871: get_num_active_clean: ceph --format xml pg dump pgs .../qa/workunits/ceph-helpers.sh:872: get_num_active_clean: xmlstarlet sel -t -m '//pg_stat/state[(contains(.,'\''active'\'') and contains(.,'\''clean'\'') and not(contains(.,'\''stale'\'')))]' -v . -n .../qa/workunits/ceph-helpers.sh:873: get_num_active_clean: grep -v '^$' .../qa/workunits/ceph-helpers.sh:873: get_num_active_clean: wc -l ../qa/workunits/ceph-helpers.sh:954: is_clean: test 0 = 4 ../qa/workunits/ceph-helpers.sh:954: is_clean: return 1 ../qa/workunits/ceph-helpers.sh:986: wait_for_clean: get_is_making_recovery_progress .../qa/workunits/ceph-helpers.sh:839: get_is_making_recovery_progress: ceph --format xml status .../qa/workunits/ceph-helpers.sh:839: get_is_making_recovery_progress: xmlstarlet sel -t -m //pgmap/recovering_keys_per_sec -v . -o ' ' -t -m //pgmap/recovering_bytes_per_sec -v . -o ' ' -t -m //pgmap/recovering_objects_per_sec -v . ../qa/workunits/ceph-helpers.sh:839: get_is_making_recovery_progress: local progress= ../qa/workunits/ceph-helpers.sh:840: get_is_making_recovery_progress: test -n '' ../qa/workunits/ceph-helpers.sh:988: wait_for_clean: (( timer >= 120 )) .../qa/workunits/ceph-helpers.sh:993: wait_for_clean: get_num_active_clean .../qa/workunits/ceph-helpers.sh:863: get_num_active_clean: local 'expression=(' .../qa/workunits/ceph-helpers.sh:864: get_num_active_clean: expression+='contains(.,'\''active'\'') and ' .../qa/workunits/ceph-helpers.sh:865: get_num_active_clean: expression+='contains(.,'\''clean'\'') and ' .../qa/workunits/ceph-helpers.sh:866: get_num_active_clean: expression+='not(contains(.,'\''stale'\''))' .../qa/workunits/ceph-helpers.sh:867: get_num_active_clean: expression+=')' .../qa/workunits/ceph-helpers.sh:871: get_num_active_clean: ceph --format xml pg dump pgs .../qa/workunits/ceph-helpers.sh:872: get_num_active_clean: xmlstarlet sel -t -m '//pg_stat/state[(contains(.,'\''active'\'') and contains(.,'\''clean'\'') and not(contains(.,'\''stale'\'')))]' -v . -n .../qa/workunits/ceph-helpers.sh:873: get_num_active_clean: grep -v '^$' .../qa/workunits/ceph-helpers.sh:873: get_num_active_clean: wc -l ../qa/workunits/ceph-helpers.sh:993: wait_for_clean: cur_active_clean=0 ../qa/workunits/ceph-helpers.sh:994: wait_for_clean: test 0 '!=' 0 ../qa/workunits/ceph-helpers.sh:998: wait_for_clean: sleep 1 .../qa/workunits/ceph-helpers.sh:999: wait_for_clean: expr 103 + 1 ../qa/workunits/ceph-helpers.sh:999: wait_for_clean: timer=104 ../qa/workunits/ceph-helpers.sh:985: wait_for_clean: is_clean .../qa/workunits/ceph-helpers.sh:952: is_clean: get_num_pgs .../qa/workunits/ceph-helpers.sh:898: get_num_pgs: ceph --format xml status .../qa/workunits/ceph-helpers.sh:899: get_num_pgs: xmlstarlet sel -t -m //pgmap/num_pgs -v . ../qa/workunits/ceph-helpers.sh:952: is_clean: num_pgs=4 ../qa/workunits/ceph-helpers.sh:953: is_clean: test 4 '!=' 0 .../qa/workunits/ceph-helpers.sh:954: is_clean: get_num_active_clean .../qa/workunits/ceph-helpers.sh:863: get_num_active_clean: local 'expression=(' .../qa/workunits/ceph-helpers.sh:864: get_num_active_clean: expression+='contains(.,'\''active'\'') and ' .../qa/workunits/ceph-helpers.sh:865: get_num_active_clean: expression+='contains(.,'\''clean'\'') and ' .../qa/workunits/ceph-helpers.sh:866: get_num_active_clean: expression+='not(contains(.,'\''stale'\''))' .../qa/workunits/ceph-helpers.sh:867: get_num_active_clean: expression+=')' .../qa/workunits/ceph-helpers.sh:871: get_num_active_clean: ceph --format xml pg dump pgs .../qa/workunits/ceph-helpers.sh:872: get_num_active_clean: xmlstarlet sel -t -m '//pg_stat/state[(contains(.,'\''active'\'') and contains(.,'\''clean'\'') and not(contains(.,'\''stale'\'')))]' -v . -n .../qa/workunits/ceph-helpers.sh:873: get_num_active_clean: wc -l .../qa/workunits/ceph-helpers.sh:873: get_num_active_clean: grep -v '^$' ../qa/workunits/ceph-helpers.sh:954: is_clean: test 0 = 4 ../qa/workunits/ceph-helpers.sh:954: is_clean: return 1 ../qa/workunits/ceph-helpers.sh:986: wait_for_clean: get_is_making_recovery_progress .../qa/workunits/ceph-helpers.sh:839: get_is_making_recovery_progress: ceph --format xml status .../qa/workunits/ceph-helpers.sh:839: get_is_making_recovery_progress: xmlstarlet sel -t -m //pgmap/recovering_keys_per_sec -v . -o ' ' -t -m //pgmap/recovering_bytes_per_sec -v . -o ' ' -t -m //pgmap/recovering_objects_per_sec -v . ../qa/workunits/ceph-helpers.sh:839: get_is_making_recovery_progress: local progress= ../qa/workunits/ceph-helpers.sh:840: get_is_making_recovery_progress: test -n '' ../qa/workunits/ceph-helpers.sh:988: wait_for_clean: (( timer >= 120 )) .../qa/workunits/ceph-helpers.sh:993: wait_for_clean: get_num_active_clean .../qa/workunits/ceph-helpers.sh:863: get_num_active_clean: local 'expression=(' .../qa/workunits/ceph-helpers.sh:864: get_num_active_clean: expression+='contains(.,'\''active'\'') and ' .../qa/workunits/ceph-helpers.sh:865: get_num_active_clean: expression+='contains(.,'\''clean'\'') and ' .../qa/workunits/ceph-helpers.sh:866: get_num_active_clean: expression+='not(contains(.,'\''stale'\''))' .../qa/workunits/ceph-helpers.sh:867: get_num_active_clean: expression+=')' .../qa/workunits/ceph-helpers.sh:871: get_num_active_clean: ceph --format xml pg dump pgs .../qa/workunits/ceph-helpers.sh:872: get_num_active_clean: xmlstarlet sel -t -m '//pg_stat/state[(contains(.,'\''active'\'') and contains(.,'\''clean'\'') and not(contains(.,'\''stale'\'')))]' -v . -n .../qa/workunits/ceph-helpers.sh:873: get_num_active_clean: wc -l .../qa/workunits/ceph-helpers.sh:873: get_num_active_clean: grep -v '^$' ../qa/workunits/ceph-helpers.sh:993: wait_for_clean: cur_active_clean=0 ../qa/workunits/ceph-helpers.sh:994: wait_for_clean: test 0 '!=' 0 ../qa/workunits/ceph-helpers.sh:998: wait_for_clean: sleep 1 .../qa/workunits/ceph-helpers.sh:999: wait_for_clean: expr 104 + 1 ../qa/workunits/ceph-helpers.sh:999: wait_for_clean: timer=105 ../qa/workunits/ceph-helpers.sh:985: wait_for_clean: is_clean .../qa/workunits/ceph-helpers.sh:952: is_clean: get_num_pgs .../qa/workunits/ceph-helpers.sh:898: get_num_pgs: ceph --format xml status .../qa/workunits/ceph-helpers.sh:899: get_num_pgs: xmlstarlet sel -t -m //pgmap/num_pgs -v . ../qa/workunits/ceph-helpers.sh:952: is_clean: num_pgs=4 ../qa/workunits/ceph-helpers.sh:953: is_clean: test 4 '!=' 0 .../qa/workunits/ceph-helpers.sh:954: is_clean: get_num_active_clean .../qa/workunits/ceph-helpers.sh:863: get_num_active_clean: local 'expression=(' .../qa/workunits/ceph-helpers.sh:864: get_num_active_clean: expression+='contains(.,'\''active'\'') and ' .../qa/workunits/ceph-helpers.sh:865: get_num_active_clean: expression+='contains(.,'\''clean'\'') and ' .../qa/workunits/ceph-helpers.sh:866: get_num_active_clean: expression+='not(contains(.,'\''stale'\''))' .../qa/workunits/ceph-helpers.sh:867: get_num_active_clean: expression+=')' .../qa/workunits/ceph-helpers.sh:871: get_num_active_clean: ceph --format xml pg dump pgs .../qa/workunits/ceph-helpers.sh:872: get_num_active_clean: xmlstarlet sel -t -m '//pg_stat/state[(contains(.,'\''active'\'') and contains(.,'\''clean'\'') and not(contains(.,'\''stale'\'')))]' -v . -n .../qa/workunits/ceph-helpers.sh:873: get_num_active_clean: grep -v '^$' .../qa/workunits/ceph-helpers.sh:873: get_num_active_clean: wc -l ../qa/workunits/ceph-helpers.sh:954: is_clean: test 0 = 4 ../qa/workunits/ceph-helpers.sh:954: is_clean: return 1 ../qa/workunits/ceph-helpers.sh:986: wait_for_clean: get_is_making_recovery_progress .../qa/workunits/ceph-helpers.sh:839: get_is_making_recovery_progress: ceph --format xml status .../qa/workunits/ceph-helpers.sh:839: get_is_making_recovery_progress: xmlstarlet sel -t -m //pgmap/recovering_keys_per_sec -v . -o ' ' -t -m //pgmap/recovering_bytes_per_sec -v . -o ' ' -t -m //pgmap/recovering_objects_per_sec -v . ../qa/workunits/ceph-helpers.sh:839: get_is_making_recovery_progress: local progress= ../qa/workunits/ceph-helpers.sh:840: get_is_making_recovery_progress: test -n '' ../qa/workunits/ceph-helpers.sh:988: wait_for_clean: (( timer >= 120 )) .../qa/workunits/ceph-helpers.sh:993: wait_for_clean: get_num_active_clean .../qa/workunits/ceph-helpers.sh:863: get_num_active_clean: local 'expression=(' .../qa/workunits/ceph-helpers.sh:864: get_num_active_clean: expression+='contains(.,'\''active'\'') and ' .../qa/workunits/ceph-helpers.sh:865: get_num_active_clean: expression+='contains(.,'\''clean'\'') and ' .../qa/workunits/ceph-helpers.sh:866: get_num_active_clean: expression+='not(contains(.,'\''stale'\''))' .../qa/workunits/ceph-helpers.sh:867: get_num_active_clean: expression+=')' .../qa/workunits/ceph-helpers.sh:871: get_num_active_clean: ceph --format xml pg dump pgs .../qa/workunits/ceph-helpers.sh:872: get_num_active_clean: xmlstarlet sel -t -m '//pg_stat/state[(contains(.,'\''active'\'') and contains(.,'\''clean'\'') and not(contains(.,'\''stale'\'')))]' -v . -n .../qa/workunits/ceph-helpers.sh:873: get_num_active_clean: grep -v '^$' .../qa/workunits/ceph-helpers.sh:873: get_num_active_clean: wc -l ../qa/workunits/ceph-helpers.sh:993: wait_for_clean: cur_active_clean=0 ../qa/workunits/ceph-helpers.sh:994: wait_for_clean: test 0 '!=' 0 ../qa/workunits/ceph-helpers.sh:998: wait_for_clean: sleep 1 .../qa/workunits/ceph-helpers.sh:999: wait_for_clean: expr 105 + 1 ../qa/workunits/ceph-helpers.sh:999: wait_for_clean: timer=106 ../qa/workunits/ceph-helpers.sh:985: wait_for_clean: is_clean .../qa/workunits/ceph-helpers.sh:952: is_clean: get_num_pgs .../qa/workunits/ceph-helpers.sh:898: get_num_pgs: ceph --format xml status .../qa/workunits/ceph-helpers.sh:899: get_num_pgs: xmlstarlet sel -t -m //pgmap/num_pgs -v . ../qa/workunits/ceph-helpers.sh:952: is_clean: num_pgs=4 ../qa/workunits/ceph-helpers.sh:953: is_clean: test 4 '!=' 0 .../qa/workunits/ceph-helpers.sh:954: is_clean: get_num_active_clean .../qa/workunits/ceph-helpers.sh:863: get_num_active_clean: local 'expression=(' .../qa/workunits/ceph-helpers.sh:864: get_num_active_clean: expression+='contains(.,'\''active'\'') and ' .../qa/workunits/ceph-helpers.sh:865: get_num_active_clean: expression+='contains(.,'\''clean'\'') and ' .../qa/workunits/ceph-helpers.sh:866: get_num_active_clean: expression+='not(contains(.,'\''stale'\''))' .../qa/workunits/ceph-helpers.sh:867: get_num_active_clean: expression+=')' .../qa/workunits/ceph-helpers.sh:871: get_num_active_clean: ceph --format xml pg dump pgs .../qa/workunits/ceph-helpers.sh:872: get_num_active_clean: xmlstarlet sel -t -m '//pg_stat/state[(contains(.,'\''active'\'') and contains(.,'\''clean'\'') and not(contains(.,'\''stale'\'')))]' -v . -n .../qa/workunits/ceph-helpers.sh:873: get_num_active_clean: grep -v '^$' .../qa/workunits/ceph-helpers.sh:873: get_num_active_clean: wc -l ../qa/workunits/ceph-helpers.sh:954: is_clean: test 0 = 4 ../qa/workunits/ceph-helpers.sh:954: is_clean: return 1 ../qa/workunits/ceph-helpers.sh:986: wait_for_clean: get_is_making_recovery_progress .../qa/workunits/ceph-helpers.sh:839: get_is_making_recovery_progress: ceph --format xml status .../qa/workunits/ceph-helpers.sh:839: get_is_making_recovery_progress: xmlstarlet sel -t -m //pgmap/recovering_keys_per_sec -v . -o ' ' -t -m //pgmap/recovering_bytes_per_sec -v . -o ' ' -t -m //pgmap/recovering_objects_per_sec -v . ../qa/workunits/ceph-helpers.sh:839: get_is_making_recovery_progress: local progress= ../qa/workunits/ceph-helpers.sh:840: get_is_making_recovery_progress: test -n '' ../qa/workunits/ceph-helpers.sh:988: wait_for_clean: (( timer >= 120 )) .../qa/workunits/ceph-helpers.sh:993: wait_for_clean: get_num_active_clean .../qa/workunits/ceph-helpers.sh:863: get_num_active_clean: local 'expression=(' .../qa/workunits/ceph-helpers.sh:864: get_num_active_clean: expression+='contains(.,'\''active'\'') and ' .../qa/workunits/ceph-helpers.sh:865: get_num_active_clean: expression+='contains(.,'\''clean'\'') and ' .../qa/workunits/ceph-helpers.sh:866: get_num_active_clean: expression+='not(contains(.,'\''stale'\''))' .../qa/workunits/ceph-helpers.sh:867: get_num_active_clean: expression+=')' .../qa/workunits/ceph-helpers.sh:871: get_num_active_clean: ceph --format xml pg dump pgs .../qa/workunits/ceph-helpers.sh:872: get_num_active_clean: xmlstarlet sel -t -m '//pg_stat/state[(contains(.,'\''active'\'') and contains(.,'\''clean'\'') and not(contains(.,'\''stale'\'')))]' -v . -n .../qa/workunits/ceph-helpers.sh:873: get_num_active_clean: grep -v '^$' .../qa/workunits/ceph-helpers.sh:873: get_num_active_clean: wc -l ../qa/workunits/ceph-helpers.sh:993: wait_for_clean: cur_active_clean=0 ../qa/workunits/ceph-helpers.sh:994: wait_for_clean: test 0 '!=' 0 ../qa/workunits/ceph-helpers.sh:998: wait_for_clean: sleep 1 .../qa/workunits/ceph-helpers.sh:999: wait_for_clean: expr 106 + 1 ../qa/workunits/ceph-helpers.sh:999: wait_for_clean: timer=107 ../qa/workunits/ceph-helpers.sh:985: wait_for_clean: is_clean .../qa/workunits/ceph-helpers.sh:952: is_clean: get_num_pgs .../qa/workunits/ceph-helpers.sh:898: get_num_pgs: ceph --format xml status .../qa/workunits/ceph-helpers.sh:899: get_num_pgs: xmlstarlet sel -t -m //pgmap/num_pgs -v . ../qa/workunits/ceph-helpers.sh:952: is_clean: num_pgs=4 ../qa/workunits/ceph-helpers.sh:953: is_clean: test 4 '!=' 0 .../qa/workunits/ceph-helpers.sh:954: is_clean: get_num_active_clean .../qa/workunits/ceph-helpers.sh:863: get_num_active_clean: local 'expression=(' .../qa/workunits/ceph-helpers.sh:864: get_num_active_clean: expression+='contains(.,'\''active'\'') and ' .../qa/workunits/ceph-helpers.sh:865: get_num_active_clean: expression+='contains(.,'\''clean'\'') and ' .../qa/workunits/ceph-helpers.sh:866: get_num_active_clean: expression+='not(contains(.,'\''stale'\''))' .../qa/workunits/ceph-helpers.sh:867: get_num_active_clean: expression+=')' .../qa/workunits/ceph-helpers.sh:871: get_num_active_clean: ceph --format xml pg dump pgs .../qa/workunits/ceph-helpers.sh:872: get_num_active_clean: xmlstarlet sel -t -m '//pg_stat/state[(contains(.,'\''active'\'') and contains(.,'\''clean'\'') and not(contains(.,'\''stale'\'')))]' -v . -n .../qa/workunits/ceph-helpers.sh:873: get_num_active_clean: grep -v '^$' .../qa/workunits/ceph-helpers.sh:873: get_num_active_clean: wc -l ../qa/workunits/ceph-helpers.sh:954: is_clean: test 0 = 4 ../qa/workunits/ceph-helpers.sh:954: is_clean: return 1 ../qa/workunits/ceph-helpers.sh:986: wait_for_clean: get_is_making_recovery_progress .../qa/workunits/ceph-helpers.sh:839: get_is_making_recovery_progress: ceph --format xml status .../qa/workunits/ceph-helpers.sh:839: get_is_making_recovery_progress: xmlstarlet sel -t -m //pgmap/recovering_keys_per_sec -v . -o ' ' -t -m //pgmap/recovering_bytes_per_sec -v . -o ' ' -t -m //pgmap/recovering_objects_per_sec -v . ../qa/workunits/ceph-helpers.sh:839: get_is_making_recovery_progress: local progress= ../qa/workunits/ceph-helpers.sh:840: get_is_making_recovery_progress: test -n '' ../qa/workunits/ceph-helpers.sh:988: wait_for_clean: (( timer >= 120 )) .../qa/workunits/ceph-helpers.sh:993: wait_for_clean: get_num_active_clean .../qa/workunits/ceph-helpers.sh:863: get_num_active_clean: local 'expression=(' .../qa/workunits/ceph-helpers.sh:864: get_num_active_clean: expression+='contains(.,'\''active'\'') and ' .../qa/workunits/ceph-helpers.sh:865: get_num_active_clean: expression+='contains(.,'\''clean'\'') and ' .../qa/workunits/ceph-helpers.sh:866: get_num_active_clean: expression+='not(contains(.,'\''stale'\''))' .../qa/workunits/ceph-helpers.sh:867: get_num_active_clean: expression+=')' .../qa/workunits/ceph-helpers.sh:871: get_num_active_clean: ceph --format xml pg dump pgs .../qa/workunits/ceph-helpers.sh:872: get_num_active_clean: xmlstarlet sel -t -m '//pg_stat/state[(contains(.,'\''active'\'') and contains(.,'\''clean'\'') and not(contains(.,'\''stale'\'')))]' -v . -n .../qa/workunits/ceph-helpers.sh:873: get_num_active_clean: grep -v '^$' .../qa/workunits/ceph-helpers.sh:873: get_num_active_clean: wc -l ../qa/workunits/ceph-helpers.sh:993: wait_for_clean: cur_active_clean=0 ../qa/workunits/ceph-helpers.sh:994: wait_for_clean: test 0 '!=' 0 ../qa/workunits/ceph-helpers.sh:998: wait_for_clean: sleep 1 .../qa/workunits/ceph-helpers.sh:999: wait_for_clean: expr 107 + 1 ../qa/workunits/ceph-helpers.sh:999: wait_for_clean: timer=108 ../qa/workunits/ceph-helpers.sh:985: wait_for_clean: is_clean .../qa/workunits/ceph-helpers.sh:952: is_clean: get_num_pgs .../qa/workunits/ceph-helpers.sh:898: get_num_pgs: ceph --format xml status .../qa/workunits/ceph-helpers.sh:899: get_num_pgs: xmlstarlet sel -t -m //pgmap/num_pgs -v . ../qa/workunits/ceph-helpers.sh:952: is_clean: num_pgs=4 ../qa/workunits/ceph-helpers.sh:953: is_clean: test 4 '!=' 0 .../qa/workunits/ceph-helpers.sh:954: is_clean: get_num_active_clean .../qa/workunits/ceph-helpers.sh:863: get_num_active_clean: local 'expression=(' .../qa/workunits/ceph-helpers.sh:864: get_num_active_clean: expression+='contains(.,'\''active'\'') and ' .../qa/workunits/ceph-helpers.sh:865: get_num_active_clean: expression+='contains(.,'\''clean'\'') and ' .../qa/workunits/ceph-helpers.sh:866: get_num_active_clean: expression+='not(contains(.,'\''stale'\''))' .../qa/workunits/ceph-helpers.sh:867: get_num_active_clean: expression+=')' .../qa/workunits/ceph-helpers.sh:871: get_num_active_clean: ceph --format xml pg dump pgs .../qa/workunits/ceph-helpers.sh:872: get_num_active_clean: xmlstarlet sel -t -m '//pg_stat/state[(contains(.,'\''active'\'') and contains(.,'\''clean'\'') and not(contains(.,'\''stale'\'')))]' -v . -n .../qa/workunits/ceph-helpers.sh:873: get_num_active_clean: grep -v '^$' .../qa/workunits/ceph-helpers.sh:873: get_num_active_clean: wc -l ../qa/workunits/ceph-helpers.sh:954: is_clean: test 0 = 4 ../qa/workunits/ceph-helpers.sh:954: is_clean: return 1 ../qa/workunits/ceph-helpers.sh:986: wait_for_clean: get_is_making_recovery_progress .../qa/workunits/ceph-helpers.sh:839: get_is_making_recovery_progress: ceph --format xml status .../qa/workunits/ceph-helpers.sh:839: get_is_making_recovery_progress: xmlstarlet sel -t -m //pgmap/recovering_keys_per_sec -v . -o ' ' -t -m //pgmap/recovering_bytes_per_sec -v . -o ' ' -t -m //pgmap/recovering_objects_per_sec -v . ../qa/workunits/ceph-helpers.sh:839: get_is_making_recovery_progress: local progress= ../qa/workunits/ceph-helpers.sh:840: get_is_making_recovery_progress: test -n '' ../qa/workunits/ceph-helpers.sh:988: wait_for_clean: (( timer >= 120 )) .../qa/workunits/ceph-helpers.sh:993: wait_for_clean: get_num_active_clean .../qa/workunits/ceph-helpers.sh:863: get_num_active_clean: local 'expression=(' .../qa/workunits/ceph-helpers.sh:864: get_num_active_clean: expression+='contains(.,'\''active'\'') and ' .../qa/workunits/ceph-helpers.sh:865: get_num_active_clean: expression+='contains(.,'\''clean'\'') and ' .../qa/workunits/ceph-helpers.sh:866: get_num_active_clean: expression+='not(contains(.,'\''stale'\''))' .../qa/workunits/ceph-helpers.sh:867: get_num_active_clean: expression+=')' .../qa/workunits/ceph-helpers.sh:871: get_num_active_clean: ceph --format xml pg dump pgs .../qa/workunits/ceph-helpers.sh:872: get_num_active_clean: xmlstarlet sel -t -m '//pg_stat/state[(contains(.,'\''active'\'') and contains(.,'\''clean'\'') and not(contains(.,'\''stale'\'')))]' -v . -n .../qa/workunits/ceph-helpers.sh:873: get_num_active_clean: grep -v '^$' .../qa/workunits/ceph-helpers.sh:873: get_num_active_clean: wc -l ../qa/workunits/ceph-helpers.sh:993: wait_for_clean: cur_active_clean=0 ../qa/workunits/ceph-helpers.sh:994: wait_for_clean: test 0 '!=' 0 ../qa/workunits/ceph-helpers.sh:998: wait_for_clean: sleep 1 .../qa/workunits/ceph-helpers.sh:999: wait_for_clean: expr 108 + 1 ../qa/workunits/ceph-helpers.sh:999: wait_for_clean: timer=109 ../qa/workunits/ceph-helpers.sh:985: wait_for_clean: is_clean .../qa/workunits/ceph-helpers.sh:952: is_clean: get_num_pgs .../qa/workunits/ceph-helpers.sh:898: get_num_pgs: ceph --format xml status .../qa/workunits/ceph-helpers.sh:899: get_num_pgs: xmlstarlet sel -t -m //pgmap/num_pgs -v . ../qa/workunits/ceph-helpers.sh:952: is_clean: num_pgs=4 ../qa/workunits/ceph-helpers.sh:953: is_clean: test 4 '!=' 0 .../qa/workunits/ceph-helpers.sh:954: is_clean: get_num_active_clean .../qa/workunits/ceph-helpers.sh:863: get_num_active_clean: local 'expression=(' .../qa/workunits/ceph-helpers.sh:864: get_num_active_clean: expression+='contains(.,'\''active'\'') and ' .../qa/workunits/ceph-helpers.sh:865: get_num_active_clean: expression+='contains(.,'\''clean'\'') and ' .../qa/workunits/ceph-helpers.sh:866: get_num_active_clean: expression+='not(contains(.,'\''stale'\''))' .../qa/workunits/ceph-helpers.sh:867: get_num_active_clean: expression+=')' .../qa/workunits/ceph-helpers.sh:871: get_num_active_clean: ceph --format xml pg dump pgs .../qa/workunits/ceph-helpers.sh:872: get_num_active_clean: xmlstarlet sel -t -m '//pg_stat/state[(contains(.,'\''active'\'') and contains(.,'\''clean'\'') and not(contains(.,'\''stale'\'')))]' -v . -n .../qa/workunits/ceph-helpers.sh:873: get_num_active_clean: grep -v '^$' .../qa/workunits/ceph-helpers.sh:873: get_num_active_clean: wc -l ../qa/workunits/ceph-helpers.sh:954: is_clean: test 0 = 4 ../qa/workunits/ceph-helpers.sh:954: is_clean: return 1 ../qa/workunits/ceph-helpers.sh:986: wait_for_clean: get_is_making_recovery_progress .../qa/workunits/ceph-helpers.sh:839: get_is_making_recovery_progress: ceph --format xml status .../qa/workunits/ceph-helpers.sh:839: get_is_making_recovery_progress: xmlstarlet sel -t -m //pgmap/recovering_keys_per_sec -v . -o ' ' -t -m //pgmap/recovering_bytes_per_sec -v . -o ' ' -t -m //pgmap/recovering_objects_per_sec -v . ../qa/workunits/ceph-helpers.sh:839: get_is_making_recovery_progress: local progress= ../qa/workunits/ceph-helpers.sh:840: get_is_making_recovery_progress: test -n '' ../qa/workunits/ceph-helpers.sh:988: wait_for_clean: (( timer >= 120 )) .../qa/workunits/ceph-helpers.sh:993: wait_for_clean: get_num_active_clean .../qa/workunits/ceph-helpers.sh:863: get_num_active_clean: local 'expression=(' .../qa/workunits/ceph-helpers.sh:864: get_num_active_clean: expression+='contains(.,'\''active'\'') and ' .../qa/workunits/ceph-helpers.sh:865: get_num_active_clean: expression+='contains(.,'\''clean'\'') and ' .../qa/workunits/ceph-helpers.sh:866: get_num_active_clean: expression+='not(contains(.,'\''stale'\''))' .../qa/workunits/ceph-helpers.sh:867: get_num_active_clean: expression+=')' .../qa/workunits/ceph-helpers.sh:871: get_num_active_clean: ceph --format xml pg dump pgs .../qa/workunits/ceph-helpers.sh:872: get_num_active_clean: xmlstarlet sel -t -m '//pg_stat/state[(contains(.,'\''active'\'') and contains(.,'\''clean'\'') and not(contains(.,'\''stale'\'')))]' -v . -n .../qa/workunits/ceph-helpers.sh:873: get_num_active_clean: wc -l .../qa/workunits/ceph-helpers.sh:873: get_num_active_clean: grep -v '^$' ../qa/workunits/ceph-helpers.sh:993: wait_for_clean: cur_active_clean=0 ../qa/workunits/ceph-helpers.sh:994: wait_for_clean: test 0 '!=' 0 ../qa/workunits/ceph-helpers.sh:998: wait_for_clean: sleep 1 .../qa/workunits/ceph-helpers.sh:999: wait_for_clean: expr 109 + 1 ../qa/workunits/ceph-helpers.sh:999: wait_for_clean: timer=110 ../qa/workunits/ceph-helpers.sh:985: wait_for_clean: is_clean .../qa/workunits/ceph-helpers.sh:952: is_clean: get_num_pgs .../qa/workunits/ceph-helpers.sh:898: get_num_pgs: ceph --format xml status .../qa/workunits/ceph-helpers.sh:899: get_num_pgs: xmlstarlet sel -t -m //pgmap/num_pgs -v . ../qa/workunits/ceph-helpers.sh:952: is_clean: num_pgs=4 ../qa/workunits/ceph-helpers.sh:953: is_clean: test 4 '!=' 0 .../qa/workunits/ceph-helpers.sh:954: is_clean: get_num_active_clean .../qa/workunits/ceph-helpers.sh:863: get_num_active_clean: local 'expression=(' .../qa/workunits/ceph-helpers.sh:864: get_num_active_clean: expression+='contains(.,'\''active'\'') and ' .../qa/workunits/ceph-helpers.sh:865: get_num_active_clean: expression+='contains(.,'\''clean'\'') and ' .../qa/workunits/ceph-helpers.sh:866: get_num_active_clean: expression+='not(contains(.,'\''stale'\''))' .../qa/workunits/ceph-helpers.sh:867: get_num_active_clean: expression+=')' .../qa/workunits/ceph-helpers.sh:871: get_num_active_clean: ceph --format xml pg dump pgs .../qa/workunits/ceph-helpers.sh:872: get_num_active_clean: xmlstarlet sel -t -m '//pg_stat/state[(contains(.,'\''active'\'') and contains(.,'\''clean'\'') and not(contains(.,'\''stale'\'')))]' -v . -n .../qa/workunits/ceph-helpers.sh:873: get_num_active_clean: grep -v '^$' .../qa/workunits/ceph-helpers.sh:873: get_num_active_clean: wc -l ../qa/workunits/ceph-helpers.sh:954: is_clean: test 0 = 4 ../qa/workunits/ceph-helpers.sh:954: is_clean: return 1 ../qa/workunits/ceph-helpers.sh:986: wait_for_clean: get_is_making_recovery_progress .../qa/workunits/ceph-helpers.sh:839: get_is_making_recovery_progress: ceph --format xml status .../qa/workunits/ceph-helpers.sh:839: get_is_making_recovery_progress: xmlstarlet sel -t -m //pgmap/recovering_keys_per_sec -v . -o ' ' -t -m //pgmap/recovering_bytes_per_sec -v . -o ' ' -t -m //pgmap/recovering_objects_per_sec -v . ../qa/workunits/ceph-helpers.sh:839: get_is_making_recovery_progress: local progress= ../qa/workunits/ceph-helpers.sh:840: get_is_making_recovery_progress: test -n '' ../qa/workunits/ceph-helpers.sh:988: wait_for_clean: (( timer >= 120 )) .../qa/workunits/ceph-helpers.sh:993: wait_for_clean: get_num_active_clean .../qa/workunits/ceph-helpers.sh:863: get_num_active_clean: local 'expression=(' .../qa/workunits/ceph-helpers.sh:864: get_num_active_clean: expression+='contains(.,'\''active'\'') and ' .../qa/workunits/ceph-helpers.sh:865: get_num_active_clean: expression+='contains(.,'\''clean'\'') and ' .../qa/workunits/ceph-helpers.sh:866: get_num_active_clean: expression+='not(contains(.,'\''stale'\''))' .../qa/workunits/ceph-helpers.sh:867: get_num_active_clean: expression+=')' .../qa/workunits/ceph-helpers.sh:871: get_num_active_clean: ceph --format xml pg dump pgs .../qa/workunits/ceph-helpers.sh:872: get_num_active_clean: xmlstarlet sel -t -m '//pg_stat/state[(contains(.,'\''active'\'') and contains(.,'\''clean'\'') and not(contains(.,'\''stale'\'')))]' -v . -n .../qa/workunits/ceph-helpers.sh:873: get_num_active_clean: grep -v '^$' .../qa/workunits/ceph-helpers.sh:873: get_num_active_clean: wc -l ../qa/workunits/ceph-helpers.sh:993: wait_for_clean: cur_active_clean=0 ../qa/workunits/ceph-helpers.sh:994: wait_for_clean: test 0 '!=' 0 ../qa/workunits/ceph-helpers.sh:998: wait_for_clean: sleep 1 .../qa/workunits/ceph-helpers.sh:999: wait_for_clean: expr 110 + 1 ../qa/workunits/ceph-helpers.sh:999: wait_for_clean: timer=111 ../qa/workunits/ceph-helpers.sh:985: wait_for_clean: is_clean .../qa/workunits/ceph-helpers.sh:952: is_clean: get_num_pgs .../qa/workunits/ceph-helpers.sh:898: get_num_pgs: ceph --format xml status .../qa/workunits/ceph-helpers.sh:899: get_num_pgs: xmlstarlet sel -t -m //pgmap/num_pgs -v . ../qa/workunits/ceph-helpers.sh:952: is_clean: num_pgs=4 ../qa/workunits/ceph-helpers.sh:953: is_clean: test 4 '!=' 0 .../qa/workunits/ceph-helpers.sh:954: is_clean: get_num_active_clean .../qa/workunits/ceph-helpers.sh:863: get_num_active_clean: local 'expression=(' .../qa/workunits/ceph-helpers.sh:864: get_num_active_clean: expression+='contains(.,'\''active'\'') and ' .../qa/workunits/ceph-helpers.sh:865: get_num_active_clean: expression+='contains(.,'\''clean'\'') and ' .../qa/workunits/ceph-helpers.sh:866: get_num_active_clean: expression+='not(contains(.,'\''stale'\''))' .../qa/workunits/ceph-helpers.sh:867: get_num_active_clean: expression+=')' .../qa/workunits/ceph-helpers.sh:871: get_num_active_clean: ceph --format xml pg dump pgs .../qa/workunits/ceph-helpers.sh:872: get_num_active_clean: xmlstarlet sel -t -m '//pg_stat/state[(contains(.,'\''active'\'') and contains(.,'\''clean'\'') and not(contains(.,'\''stale'\'')))]' -v . -n .../qa/workunits/ceph-helpers.sh:873: get_num_active_clean: grep -v '^$' .../qa/workunits/ceph-helpers.sh:873: get_num_active_clean: wc -l ../qa/workunits/ceph-helpers.sh:954: is_clean: test 0 = 4 ../qa/workunits/ceph-helpers.sh:954: is_clean: return 1 ../qa/workunits/ceph-helpers.sh:986: wait_for_clean: get_is_making_recovery_progress .../qa/workunits/ceph-helpers.sh:839: get_is_making_recovery_progress: ceph --format xml status .../qa/workunits/ceph-helpers.sh:839: get_is_making_recovery_progress: xmlstarlet sel -t -m //pgmap/recovering_keys_per_sec -v . -o ' ' -t -m //pgmap/recovering_bytes_per_sec -v . -o ' ' -t -m //pgmap/recovering_objects_per_sec -v . ../qa/workunits/ceph-helpers.sh:839: get_is_making_recovery_progress: local progress= ../qa/workunits/ceph-helpers.sh:840: get_is_making_recovery_progress: test -n '' ../qa/workunits/ceph-helpers.sh:988: wait_for_clean: (( timer >= 120 )) .../qa/workunits/ceph-helpers.sh:993: wait_for_clean: get_num_active_clean .../qa/workunits/ceph-helpers.sh:863: get_num_active_clean: local 'expression=(' .../qa/workunits/ceph-helpers.sh:864: get_num_active_clean: expression+='contains(.,'\''active'\'') and ' .../qa/workunits/ceph-helpers.sh:865: get_num_active_clean: expression+='contains(.,'\''clean'\'') and ' .../qa/workunits/ceph-helpers.sh:866: get_num_active_clean: expression+='not(contains(.,'\''stale'\''))' .../qa/workunits/ceph-helpers.sh:867: get_num_active_clean: expression+=')' .../qa/workunits/ceph-helpers.sh:871: get_num_active_clean: ceph --format xml pg dump pgs .../qa/workunits/ceph-helpers.sh:872: get_num_active_clean: xmlstarlet sel -t -m '//pg_stat/state[(contains(.,'\''active'\'') and contains(.,'\''clean'\'') and not(contains(.,'\''stale'\'')))]' -v . -n .../qa/workunits/ceph-helpers.sh:873: get_num_active_clean: grep -v '^$' .../qa/workunits/ceph-helpers.sh:873: get_num_active_clean: wc -l ../qa/workunits/ceph-helpers.sh:993: wait_for_clean: cur_active_clean=0 ../qa/workunits/ceph-helpers.sh:994: wait_for_clean: test 0 '!=' 0 ../qa/workunits/ceph-helpers.sh:998: wait_for_clean: sleep 1 .../qa/workunits/ceph-helpers.sh:999: wait_for_clean: expr 111 + 1 ../qa/workunits/ceph-helpers.sh:999: wait_for_clean: timer=112 ../qa/workunits/ceph-helpers.sh:985: wait_for_clean: is_clean .../qa/workunits/ceph-helpers.sh:952: is_clean: get_num_pgs .../qa/workunits/ceph-helpers.sh:898: get_num_pgs: ceph --format xml status .../qa/workunits/ceph-helpers.sh:899: get_num_pgs: xmlstarlet sel -t -m //pgmap/num_pgs -v . ../qa/workunits/ceph-helpers.sh:952: is_clean: num_pgs=4 ../qa/workunits/ceph-helpers.sh:953: is_clean: test 4 '!=' 0 .../qa/workunits/ceph-helpers.sh:954: is_clean: get_num_active_clean .../qa/workunits/ceph-helpers.sh:863: get_num_active_clean: local 'expression=(' .../qa/workunits/ceph-helpers.sh:864: get_num_active_clean: expression+='contains(.,'\''active'\'') and ' .../qa/workunits/ceph-helpers.sh:865: get_num_active_clean: expression+='contains(.,'\''clean'\'') and ' .../qa/workunits/ceph-helpers.sh:866: get_num_active_clean: expression+='not(contains(.,'\''stale'\''))' .../qa/workunits/ceph-helpers.sh:867: get_num_active_clean: expression+=')' .../qa/workunits/ceph-helpers.sh:871: get_num_active_clean: ceph --format xml pg dump pgs .../qa/workunits/ceph-helpers.sh:872: get_num_active_clean: xmlstarlet sel -t -m '//pg_stat/state[(contains(.,'\''active'\'') and contains(.,'\''clean'\'') and not(contains(.,'\''stale'\'')))]' -v . -n .../qa/workunits/ceph-helpers.sh:873: get_num_active_clean: wc -l .../qa/workunits/ceph-helpers.sh:873: get_num_active_clean: grep -v '^$' ../qa/workunits/ceph-helpers.sh:954: is_clean: test 0 = 4 ../qa/workunits/ceph-helpers.sh:954: is_clean: return 1 ../qa/workunits/ceph-helpers.sh:986: wait_for_clean: get_is_making_recovery_progress .../qa/workunits/ceph-helpers.sh:839: get_is_making_recovery_progress: ceph --format xml status .../qa/workunits/ceph-helpers.sh:839: get_is_making_recovery_progress: xmlstarlet sel -t -m //pgmap/recovering_keys_per_sec -v . -o ' ' -t -m //pgmap/recovering_bytes_per_sec -v . -o ' ' -t -m //pgmap/recovering_objects_per_sec -v . ../qa/workunits/ceph-helpers.sh:839: get_is_making_recovery_progress: local progress= ../qa/workunits/ceph-helpers.sh:840: get_is_making_recovery_progress: test -n '' ../qa/workunits/ceph-helpers.sh:988: wait_for_clean: (( timer >= 120 )) .../qa/workunits/ceph-helpers.sh:993: wait_for_clean: get_num_active_clean .../qa/workunits/ceph-helpers.sh:863: get_num_active_clean: local 'expression=(' .../qa/workunits/ceph-helpers.sh:864: get_num_active_clean: expression+='contains(.,'\''active'\'') and ' .../qa/workunits/ceph-helpers.sh:865: get_num_active_clean: expression+='contains(.,'\''clean'\'') and ' .../qa/workunits/ceph-helpers.sh:866: get_num_active_clean: expression+='not(contains(.,'\''stale'\''))' .../qa/workunits/ceph-helpers.sh:867: get_num_active_clean: expression+=')' .../qa/workunits/ceph-helpers.sh:871: get_num_active_clean: ceph --format xml pg dump pgs .../qa/workunits/ceph-helpers.sh:872: get_num_active_clean: xmlstarlet sel -t -m '//pg_stat/state[(contains(.,'\''active'\'') and contains(.,'\''clean'\'') and not(contains(.,'\''stale'\'')))]' -v . -n .../qa/workunits/ceph-helpers.sh:873: get_num_active_clean: wc -l .../qa/workunits/ceph-helpers.sh:873: get_num_active_clean: grep -v '^$' ../qa/workunits/ceph-helpers.sh:993: wait_for_clean: cur_active_clean=0 ../qa/workunits/ceph-helpers.sh:994: wait_for_clean: test 0 '!=' 0 ../qa/workunits/ceph-helpers.sh:998: wait_for_clean: sleep 1 .../qa/workunits/ceph-helpers.sh:999: wait_for_clean: expr 112 + 1 ../qa/workunits/ceph-helpers.sh:999: wait_for_clean: timer=113 ../qa/workunits/ceph-helpers.sh:985: wait_for_clean: is_clean .../qa/workunits/ceph-helpers.sh:952: is_clean: get_num_pgs .../qa/workunits/ceph-helpers.sh:898: get_num_pgs: ceph --format xml status .../qa/workunits/ceph-helpers.sh:899: get_num_pgs: xmlstarlet sel -t -m //pgmap/num_pgs -v . ../qa/workunits/ceph-helpers.sh:952: is_clean: num_pgs=4 ../qa/workunits/ceph-helpers.sh:953: is_clean: test 4 '!=' 0 .../qa/workunits/ceph-helpers.sh:954: is_clean: get_num_active_clean .../qa/workunits/ceph-helpers.sh:863: get_num_active_clean: local 'expression=(' .../qa/workunits/ceph-helpers.sh:864: get_num_active_clean: expression+='contains(.,'\''active'\'') and ' .../qa/workunits/ceph-helpers.sh:865: get_num_active_clean: expression+='contains(.,'\''clean'\'') and ' .../qa/workunits/ceph-helpers.sh:866: get_num_active_clean: expression+='not(contains(.,'\''stale'\''))' .../qa/workunits/ceph-helpers.sh:867: get_num_active_clean: expression+=')' .../qa/workunits/ceph-helpers.sh:871: get_num_active_clean: ceph --format xml pg dump pgs .../qa/workunits/ceph-helpers.sh:872: get_num_active_clean: xmlstarlet sel -t -m '//pg_stat/state[(contains(.,'\''active'\'') and contains(.,'\''clean'\'') and not(contains(.,'\''stale'\'')))]' -v . -n .../qa/workunits/ceph-helpers.sh:873: get_num_active_clean: grep -v '^$' .../qa/workunits/ceph-helpers.sh:873: get_num_active_clean: wc -l ../qa/workunits/ceph-helpers.sh:954: is_clean: test 0 = 4 ../qa/workunits/ceph-helpers.sh:954: is_clean: return 1 ../qa/workunits/ceph-helpers.sh:986: wait_for_clean: get_is_making_recovery_progress .../qa/workunits/ceph-helpers.sh:839: get_is_making_recovery_progress: ceph --format xml status .../qa/workunits/ceph-helpers.sh:839: get_is_making_recovery_progress: xmlstarlet sel -t -m //pgmap/recovering_keys_per_sec -v . -o ' ' -t -m //pgmap/recovering_bytes_per_sec -v . -o ' ' -t -m //pgmap/recovering_objects_per_sec -v . ../qa/workunits/ceph-helpers.sh:839: get_is_making_recovery_progress: local progress= ../qa/workunits/ceph-helpers.sh:840: get_is_making_recovery_progress: test -n '' ../qa/workunits/ceph-helpers.sh:988: wait_for_clean: (( timer >= 120 )) .../qa/workunits/ceph-helpers.sh:993: wait_for_clean: get_num_active_clean .../qa/workunits/ceph-helpers.sh:863: get_num_active_clean: local 'expression=(' .../qa/workunits/ceph-helpers.sh:864: get_num_active_clean: expression+='contains(.,'\''active'\'') and ' .../qa/workunits/ceph-helpers.sh:865: get_num_active_clean: expression+='contains(.,'\''clean'\'') and ' .../qa/workunits/ceph-helpers.sh:866: get_num_active_clean: expression+='not(contains(.,'\''stale'\''))' .../qa/workunits/ceph-helpers.sh:867: get_num_active_clean: expression+=')' .../qa/workunits/ceph-helpers.sh:871: get_num_active_clean: ceph --format xml pg dump pgs .../qa/workunits/ceph-helpers.sh:872: get_num_active_clean: xmlstarlet sel -t -m '//pg_stat/state[(contains(.,'\''active'\'') and contains(.,'\''clean'\'') and not(contains(.,'\''stale'\'')))]' -v . -n .../qa/workunits/ceph-helpers.sh:873: get_num_active_clean: grep -v '^$' .../qa/workunits/ceph-helpers.sh:873: get_num_active_clean: wc -l ../qa/workunits/ceph-helpers.sh:993: wait_for_clean: cur_active_clean=0 ../qa/workunits/ceph-helpers.sh:994: wait_for_clean: test 0 '!=' 0 ../qa/workunits/ceph-helpers.sh:998: wait_for_clean: sleep 1 .../qa/workunits/ceph-helpers.sh:999: wait_for_clean: expr 113 + 1 ../qa/workunits/ceph-helpers.sh:999: wait_for_clean: timer=114 ../qa/workunits/ceph-helpers.sh:985: wait_for_clean: is_clean .../qa/workunits/ceph-helpers.sh:952: is_clean: get_num_pgs .../qa/workunits/ceph-helpers.sh:898: get_num_pgs: ceph --format xml status .../qa/workunits/ceph-helpers.sh:899: get_num_pgs: xmlstarlet sel -t -m //pgmap/num_pgs -v . ../qa/workunits/ceph-helpers.sh:952: is_clean: num_pgs=4 ../qa/workunits/ceph-helpers.sh:953: is_clean: test 4 '!=' 0 .../qa/workunits/ceph-helpers.sh:954: is_clean: get_num_active_clean .../qa/workunits/ceph-helpers.sh:863: get_num_active_clean: local 'expression=(' .../qa/workunits/ceph-helpers.sh:864: get_num_active_clean: expression+='contains(.,'\''active'\'') and ' .../qa/workunits/ceph-helpers.sh:865: get_num_active_clean: expression+='contains(.,'\''clean'\'') and ' .../qa/workunits/ceph-helpers.sh:866: get_num_active_clean: expression+='not(contains(.,'\''stale'\''))' .../qa/workunits/ceph-helpers.sh:867: get_num_active_clean: expression+=')' .../qa/workunits/ceph-helpers.sh:871: get_num_active_clean: ceph --format xml pg dump pgs .../qa/workunits/ceph-helpers.sh:872: get_num_active_clean: xmlstarlet sel -t -m '//pg_stat/state[(contains(.,'\''active'\'') and contains(.,'\''clean'\'') and not(contains(.,'\''stale'\'')))]' -v . -n .../qa/workunits/ceph-helpers.sh:873: get_num_active_clean: grep -v '^$' .../qa/workunits/ceph-helpers.sh:873: get_num_active_clean: wc -l ../qa/workunits/ceph-helpers.sh:954: is_clean: test 0 = 4 ../qa/workunits/ceph-helpers.sh:954: is_clean: return 1 ../qa/workunits/ceph-helpers.sh:986: wait_for_clean: get_is_making_recovery_progress .../qa/workunits/ceph-helpers.sh:839: get_is_making_recovery_progress: ceph --format xml status .../qa/workunits/ceph-helpers.sh:839: get_is_making_recovery_progress: xmlstarlet sel -t -m //pgmap/recovering_keys_per_sec -v . -o ' ' -t -m //pgmap/recovering_bytes_per_sec -v . -o ' ' -t -m //pgmap/recovering_objects_per_sec -v . ../qa/workunits/ceph-helpers.sh:839: get_is_making_recovery_progress: local progress= ../qa/workunits/ceph-helpers.sh:840: get_is_making_recovery_progress: test -n '' ../qa/workunits/ceph-helpers.sh:988: wait_for_clean: (( timer >= 120 )) .../qa/workunits/ceph-helpers.sh:993: wait_for_clean: get_num_active_clean .../qa/workunits/ceph-helpers.sh:863: get_num_active_clean: local 'expression=(' .../qa/workunits/ceph-helpers.sh:864: get_num_active_clean: expression+='contains(.,'\''active'\'') and ' .../qa/workunits/ceph-helpers.sh:865: get_num_active_clean: expression+='contains(.,'\''clean'\'') and ' .../qa/workunits/ceph-helpers.sh:866: get_num_active_clean: expression+='not(contains(.,'\''stale'\''))' .../qa/workunits/ceph-helpers.sh:867: get_num_active_clean: expression+=')' .../qa/workunits/ceph-helpers.sh:871: get_num_active_clean: ceph --format xml pg dump pgs .../qa/workunits/ceph-helpers.sh:872: get_num_active_clean: xmlstarlet sel -t -m '//pg_stat/state[(contains(.,'\''active'\'') and contains(.,'\''clean'\'') and not(contains(.,'\''stale'\'')))]' -v . -n .../qa/workunits/ceph-helpers.sh:873: get_num_active_clean: grep -v '^$' .../qa/workunits/ceph-helpers.sh:873: get_num_active_clean: wc -l ../qa/workunits/ceph-helpers.sh:993: wait_for_clean: cur_active_clean=0 ../qa/workunits/ceph-helpers.sh:994: wait_for_clean: test 0 '!=' 0 ../qa/workunits/ceph-helpers.sh:998: wait_for_clean: sleep 1 .../qa/workunits/ceph-helpers.sh:999: wait_for_clean: expr 114 + 1 ../qa/workunits/ceph-helpers.sh:999: wait_for_clean: timer=115 ../qa/workunits/ceph-helpers.sh:985: wait_for_clean: is_clean .../qa/workunits/ceph-helpers.sh:952: is_clean: get_num_pgs .../qa/workunits/ceph-helpers.sh:898: get_num_pgs: ceph --format xml status .../qa/workunits/ceph-helpers.sh:899: get_num_pgs: xmlstarlet sel -t -m //pgmap/num_pgs -v . ../qa/workunits/ceph-helpers.sh:952: is_clean: num_pgs=4 ../qa/workunits/ceph-helpers.sh:953: is_clean: test 4 '!=' 0 .../qa/workunits/ceph-helpers.sh:954: is_clean: get_num_active_clean .../qa/workunits/ceph-helpers.sh:863: get_num_active_clean: local 'expression=(' .../qa/workunits/ceph-helpers.sh:864: get_num_active_clean: expression+='contains(.,'\''active'\'') and ' .../qa/workunits/ceph-helpers.sh:865: get_num_active_clean: expression+='contains(.,'\''clean'\'') and ' .../qa/workunits/ceph-helpers.sh:866: get_num_active_clean: expression+='not(contains(.,'\''stale'\''))' .../qa/workunits/ceph-helpers.sh:867: get_num_active_clean: expression+=')' .../qa/workunits/ceph-helpers.sh:871: get_num_active_clean: ceph --format xml pg dump pgs .../qa/workunits/ceph-helpers.sh:872: get_num_active_clean: xmlstarlet sel -t -m '//pg_stat/state[(contains(.,'\''active'\'') and contains(.,'\''clean'\'') and not(contains(.,'\''stale'\'')))]' -v . -n .../qa/workunits/ceph-helpers.sh:873: get_num_active_clean: wc -l .../qa/workunits/ceph-helpers.sh:873: get_num_active_clean: grep -v '^$' ../qa/workunits/ceph-helpers.sh:954: is_clean: test 0 = 4 ../qa/workunits/ceph-helpers.sh:954: is_clean: return 1 ../qa/workunits/ceph-helpers.sh:986: wait_for_clean: get_is_making_recovery_progress .../qa/workunits/ceph-helpers.sh:839: get_is_making_recovery_progress: ceph --format xml status .../qa/workunits/ceph-helpers.sh:839: get_is_making_recovery_progress: xmlstarlet sel -t -m //pgmap/recovering_keys_per_sec -v . -o ' ' -t -m //pgmap/recovering_bytes_per_sec -v . -o ' ' -t -m //pgmap/recovering_objects_per_sec -v . ../qa/workunits/ceph-helpers.sh:839: get_is_making_recovery_progress: local progress= ../qa/workunits/ceph-helpers.sh:840: get_is_making_recovery_progress: test -n '' ../qa/workunits/ceph-helpers.sh:988: wait_for_clean: (( timer >= 120 )) .../qa/workunits/ceph-helpers.sh:993: wait_for_clean: get_num_active_clean .../qa/workunits/ceph-helpers.sh:863: get_num_active_clean: local 'expression=(' .../qa/workunits/ceph-helpers.sh:864: get_num_active_clean: expression+='contains(.,'\''active'\'') and ' .../qa/workunits/ceph-helpers.sh:865: get_num_active_clean: expression+='contains(.,'\''clean'\'') and ' .../qa/workunits/ceph-helpers.sh:866: get_num_active_clean: expression+='not(contains(.,'\''stale'\''))' .../qa/workunits/ceph-helpers.sh:867: get_num_active_clean: expression+=')' .../qa/workunits/ceph-helpers.sh:871: get_num_active_clean: ceph --format xml pg dump pgs .../qa/workunits/ceph-helpers.sh:872: get_num_active_clean: xmlstarlet sel -t -m '//pg_stat/state[(contains(.,'\''active'\'') and contains(.,'\''clean'\'') and not(contains(.,'\''stale'\'')))]' -v . -n .../qa/workunits/ceph-helpers.sh:873: get_num_active_clean: grep -v '^$' .../qa/workunits/ceph-helpers.sh:873: get_num_active_clean: wc -l ../qa/workunits/ceph-helpers.sh:993: wait_for_clean: cur_active_clean=0 ../qa/workunits/ceph-helpers.sh:994: wait_for_clean: test 0 '!=' 0 ../qa/workunits/ceph-helpers.sh:998: wait_for_clean: sleep 1 .../qa/workunits/ceph-helpers.sh:999: wait_for_clean: expr 115 + 1 ../qa/workunits/ceph-helpers.sh:999: wait_for_clean: timer=116 ../qa/workunits/ceph-helpers.sh:985: wait_for_clean: is_clean .../qa/workunits/ceph-helpers.sh:952: is_clean: get_num_pgs .../qa/workunits/ceph-helpers.sh:898: get_num_pgs: ceph --format xml status .../qa/workunits/ceph-helpers.sh:899: get_num_pgs: xmlstarlet sel -t -m //pgmap/num_pgs -v . ../qa/workunits/ceph-helpers.sh:952: is_clean: num_pgs=4 ../qa/workunits/ceph-helpers.sh:953: is_clean: test 4 '!=' 0 .../qa/workunits/ceph-helpers.sh:954: is_clean: get_num_active_clean .../qa/workunits/ceph-helpers.sh:863: get_num_active_clean: local 'expression=(' .../qa/workunits/ceph-helpers.sh:864: get_num_active_clean: expression+='contains(.,'\''active'\'') and ' .../qa/workunits/ceph-helpers.sh:865: get_num_active_clean: expression+='contains(.,'\''clean'\'') and ' .../qa/workunits/ceph-helpers.sh:866: get_num_active_clean: expression+='not(contains(.,'\''stale'\''))' .../qa/workunits/ceph-helpers.sh:867: get_num_active_clean: expression+=')' .../qa/workunits/ceph-helpers.sh:871: get_num_active_clean: ceph --format xml pg dump pgs .../qa/workunits/ceph-helpers.sh:872: get_num_active_clean: xmlstarlet sel -t -m '//pg_stat/state[(contains(.,'\''active'\'') and contains(.,'\''clean'\'') and not(contains(.,'\''stale'\'')))]' -v . -n .../qa/workunits/ceph-helpers.sh:873: get_num_active_clean: grep -v '^$' .../qa/workunits/ceph-helpers.sh:873: get_num_active_clean: wc -l ../qa/workunits/ceph-helpers.sh:954: is_clean: test 0 = 4 ../qa/workunits/ceph-helpers.sh:954: is_clean: return 1 ../qa/workunits/ceph-helpers.sh:986: wait_for_clean: get_is_making_recovery_progress .../qa/workunits/ceph-helpers.sh:839: get_is_making_recovery_progress: ceph --format xml status .../qa/workunits/ceph-helpers.sh:839: get_is_making_recovery_progress: xmlstarlet sel -t -m //pgmap/recovering_keys_per_sec -v . -o ' ' -t -m //pgmap/recovering_bytes_per_sec -v . -o ' ' -t -m //pgmap/recovering_objects_per_sec -v . ../qa/workunits/ceph-helpers.sh:839: get_is_making_recovery_progress: local progress= ../qa/workunits/ceph-helpers.sh:840: get_is_making_recovery_progress: test -n '' ../qa/workunits/ceph-helpers.sh:988: wait_for_clean: (( timer >= 120 )) .../qa/workunits/ceph-helpers.sh:993: wait_for_clean: get_num_active_clean .../qa/workunits/ceph-helpers.sh:863: get_num_active_clean: local 'expression=(' .../qa/workunits/ceph-helpers.sh:864: get_num_active_clean: expression+='contains(.,'\''active'\'') and ' .../qa/workunits/ceph-helpers.sh:865: get_num_active_clean: expression+='contains(.,'\''clean'\'') and ' .../qa/workunits/ceph-helpers.sh:866: get_num_active_clean: expression+='not(contains(.,'\''stale'\''))' .../qa/workunits/ceph-helpers.sh:867: get_num_active_clean: expression+=')' .../qa/workunits/ceph-helpers.sh:871: get_num_active_clean: ceph --format xml pg dump pgs .../qa/workunits/ceph-helpers.sh:872: get_num_active_clean: xmlstarlet sel -t -m '//pg_stat/state[(contains(.,'\''active'\'') and contains(.,'\''clean'\'') and not(contains(.,'\''stale'\'')))]' -v . -n .../qa/workunits/ceph-helpers.sh:873: get_num_active_clean: grep -v '^$' .../qa/workunits/ceph-helpers.sh:873: get_num_active_clean: wc -l ../qa/workunits/ceph-helpers.sh:993: wait_for_clean: cur_active_clean=0 ../qa/workunits/ceph-helpers.sh:994: wait_for_clean: test 0 '!=' 0 ../qa/workunits/ceph-helpers.sh:998: wait_for_clean: sleep 1 .../qa/workunits/ceph-helpers.sh:999: wait_for_clean: expr 116 + 1 ../qa/workunits/ceph-helpers.sh:999: wait_for_clean: timer=117 ../qa/workunits/ceph-helpers.sh:985: wait_for_clean: is_clean .../qa/workunits/ceph-helpers.sh:952: is_clean: get_num_pgs .../qa/workunits/ceph-helpers.sh:898: get_num_pgs: ceph --format xml status .../qa/workunits/ceph-helpers.sh:899: get_num_pgs: xmlstarlet sel -t -m //pgmap/num_pgs -v . ../qa/workunits/ceph-helpers.sh:952: is_clean: num_pgs=4 ../qa/workunits/ceph-helpers.sh:953: is_clean: test 4 '!=' 0 .../qa/workunits/ceph-helpers.sh:954: is_clean: get_num_active_clean .../qa/workunits/ceph-helpers.sh:863: get_num_active_clean: local 'expression=(' .../qa/workunits/ceph-helpers.sh:864: get_num_active_clean: expression+='contains(.,'\''active'\'') and ' .../qa/workunits/ceph-helpers.sh:865: get_num_active_clean: expression+='contains(.,'\''clean'\'') and ' .../qa/workunits/ceph-helpers.sh:866: get_num_active_clean: expression+='not(contains(.,'\''stale'\''))' .../qa/workunits/ceph-helpers.sh:867: get_num_active_clean: expression+=')' .../qa/workunits/ceph-helpers.sh:871: get_num_active_clean: ceph --format xml pg dump pgs .../qa/workunits/ceph-helpers.sh:872: get_num_active_clean: xmlstarlet sel -t -m '//pg_stat/state[(contains(.,'\''active'\'') and contains(.,'\''clean'\'') and not(contains(.,'\''stale'\'')))]' -v . -n .../qa/workunits/ceph-helpers.sh:873: get_num_active_clean: grep -v '^$' .../qa/workunits/ceph-helpers.sh:873: get_num_active_clean: wc -l ../qa/workunits/ceph-helpers.sh:954: is_clean: test 0 = 4 ../qa/workunits/ceph-helpers.sh:954: is_clean: return 1 ../qa/workunits/ceph-helpers.sh:986: wait_for_clean: get_is_making_recovery_progress .../qa/workunits/ceph-helpers.sh:839: get_is_making_recovery_progress: ceph --format xml status .../qa/workunits/ceph-helpers.sh:839: get_is_making_recovery_progress: xmlstarlet sel -t -m //pgmap/recovering_keys_per_sec -v . -o ' ' -t -m //pgmap/recovering_bytes_per_sec -v . -o ' ' -t -m //pgmap/recovering_objects_per_sec -v . ../qa/workunits/ceph-helpers.sh:839: get_is_making_recovery_progress: local progress= ../qa/workunits/ceph-helpers.sh:840: get_is_making_recovery_progress: test -n '' ../qa/workunits/ceph-helpers.sh:988: wait_for_clean: (( timer >= 120 )) .../qa/workunits/ceph-helpers.sh:993: wait_for_clean: get_num_active_clean .../qa/workunits/ceph-helpers.sh:863: get_num_active_clean: local 'expression=(' .../qa/workunits/ceph-helpers.sh:864: get_num_active_clean: expression+='contains(.,'\''active'\'') and ' .../qa/workunits/ceph-helpers.sh:865: get_num_active_clean: expression+='contains(.,'\''clean'\'') and ' .../qa/workunits/ceph-helpers.sh:866: get_num_active_clean: expression+='not(contains(.,'\''stale'\''))' .../qa/workunits/ceph-helpers.sh:867: get_num_active_clean: expression+=')' .../qa/workunits/ceph-helpers.sh:871: get_num_active_clean: ceph --format xml pg dump pgs .../qa/workunits/ceph-helpers.sh:872: get_num_active_clean: xmlstarlet sel -t -m '//pg_stat/state[(contains(.,'\''active'\'') and contains(.,'\''clean'\'') and not(contains(.,'\''stale'\'')))]' -v . -n .../qa/workunits/ceph-helpers.sh:873: get_num_active_clean: grep -v '^$' .../qa/workunits/ceph-helpers.sh:873: get_num_active_clean: wc -l ../qa/workunits/ceph-helpers.sh:993: wait_for_clean: cur_active_clean=0 ../qa/workunits/ceph-helpers.sh:994: wait_for_clean: test 0 '!=' 0 ../qa/workunits/ceph-helpers.sh:998: wait_for_clean: sleep 1 .../qa/workunits/ceph-helpers.sh:999: wait_for_clean: expr 117 + 1 ../qa/workunits/ceph-helpers.sh:999: wait_for_clean: timer=118 ../qa/workunits/ceph-helpers.sh:985: wait_for_clean: is_clean .../qa/workunits/ceph-helpers.sh:952: is_clean: get_num_pgs .../qa/workunits/ceph-helpers.sh:898: get_num_pgs: ceph --format xml status .../qa/workunits/ceph-helpers.sh:899: get_num_pgs: xmlstarlet sel -t -m //pgmap/num_pgs -v . ../qa/workunits/ceph-helpers.sh:952: is_clean: num_pgs=4 ../qa/workunits/ceph-helpers.sh:953: is_clean: test 4 '!=' 0 .../qa/workunits/ceph-helpers.sh:954: is_clean: get_num_active_clean .../qa/workunits/ceph-helpers.sh:863: get_num_active_clean: local 'expression=(' .../qa/workunits/ceph-helpers.sh:864: get_num_active_clean: expression+='contains(.,'\''active'\'') and ' .../qa/workunits/ceph-helpers.sh:865: get_num_active_clean: expression+='contains(.,'\''clean'\'') and ' .../qa/workunits/ceph-helpers.sh:866: get_num_active_clean: expression+='not(contains(.,'\''stale'\''))' .../qa/workunits/ceph-helpers.sh:867: get_num_active_clean: expression+=')' .../qa/workunits/ceph-helpers.sh:872: get_num_active_clean: xmlstarlet sel -t -m '//pg_stat/state[(contains(.,'\''active'\'') and contains(.,'\''clean'\'') and not(contains(.,'\''stale'\'')))]' -v . -n .../qa/workunits/ceph-helpers.sh:873: get_num_active_clean: wc -l .../qa/workunits/ceph-helpers.sh:871: get_num_active_clean: ceph --format xml pg dump pgs .../qa/workunits/ceph-helpers.sh:873: get_num_active_clean: grep -v '^$' ../qa/workunits/ceph-helpers.sh:954: is_clean: test 0 = 4 ../qa/workunits/ceph-helpers.sh:954: is_clean: return 1 ../qa/workunits/ceph-helpers.sh:986: wait_for_clean: get_is_making_recovery_progress .../qa/workunits/ceph-helpers.sh:839: get_is_making_recovery_progress: ceph --format xml status .../qa/workunits/ceph-helpers.sh:839: get_is_making_recovery_progress: xmlstarlet sel -t -m //pgmap/recovering_keys_per_sec -v . -o ' ' -t -m //pgmap/recovering_bytes_per_sec -v . -o ' ' -t -m //pgmap/recovering_objects_per_sec -v . ../qa/workunits/ceph-helpers.sh:839: get_is_making_recovery_progress: local progress= ../qa/workunits/ceph-helpers.sh:840: get_is_making_recovery_progress: test -n '' ../qa/workunits/ceph-helpers.sh:988: wait_for_clean: (( timer >= 120 )) .../qa/workunits/ceph-helpers.sh:993: wait_for_clean: get_num_active_clean .../qa/workunits/ceph-helpers.sh:863: get_num_active_clean: local 'expression=(' .../qa/workunits/ceph-helpers.sh:864: get_num_active_clean: expression+='contains(.,'\''active'\'') and ' .../qa/workunits/ceph-helpers.sh:865: get_num_active_clean: expression+='contains(.,'\''clean'\'') and ' .../qa/workunits/ceph-helpers.sh:866: get_num_active_clean: expression+='not(contains(.,'\''stale'\''))' .../qa/workunits/ceph-helpers.sh:867: get_num_active_clean: expression+=')' .../qa/workunits/ceph-helpers.sh:871: get_num_active_clean: ceph --format xml pg dump pgs .../qa/workunits/ceph-helpers.sh:872: get_num_active_clean: xmlstarlet sel -t -m '//pg_stat/state[(contains(.,'\''active'\'') and contains(.,'\''clean'\'') and not(contains(.,'\''stale'\'')))]' -v . -n .../qa/workunits/ceph-helpers.sh:873: get_num_active_clean: grep -v '^$' .../qa/workunits/ceph-helpers.sh:873: get_num_active_clean: wc -l ../qa/workunits/ceph-helpers.sh:993: wait_for_clean: cur_active_clean=0 ../qa/workunits/ceph-helpers.sh:994: wait_for_clean: test 0 '!=' 0 ../qa/workunits/ceph-helpers.sh:998: wait_for_clean: sleep 1 .../qa/workunits/ceph-helpers.sh:999: wait_for_clean: expr 118 + 1 ../qa/workunits/ceph-helpers.sh:999: wait_for_clean: timer=119 ../qa/workunits/ceph-helpers.sh:985: wait_for_clean: is_clean .../qa/workunits/ceph-helpers.sh:952: is_clean: get_num_pgs .../qa/workunits/ceph-helpers.sh:898: get_num_pgs: ceph --format xml status .../qa/workunits/ceph-helpers.sh:899: get_num_pgs: xmlstarlet sel -t -m //pgmap/num_pgs -v . ../qa/workunits/ceph-helpers.sh:952: is_clean: num_pgs=4 ../qa/workunits/ceph-helpers.sh:953: is_clean: test 4 '!=' 0 .../qa/workunits/ceph-helpers.sh:954: is_clean: get_num_active_clean .../qa/workunits/ceph-helpers.sh:863: get_num_active_clean: local 'expression=(' .../qa/workunits/ceph-helpers.sh:864: get_num_active_clean: expression+='contains(.,'\''active'\'') and ' .../qa/workunits/ceph-helpers.sh:865: get_num_active_clean: expression+='contains(.,'\''clean'\'') and ' .../qa/workunits/ceph-helpers.sh:866: get_num_active_clean: expression+='not(contains(.,'\''stale'\''))' .../qa/workunits/ceph-helpers.sh:867: get_num_active_clean: expression+=')' .../qa/workunits/ceph-helpers.sh:871: get_num_active_clean: ceph --format xml pg dump pgs .../qa/workunits/ceph-helpers.sh:872: get_num_active_clean: xmlstarlet sel -t -m '//pg_stat/state[(contains(.,'\''active'\'') and contains(.,'\''clean'\'') and not(contains(.,'\''stale'\'')))]' -v . -n .../qa/workunits/ceph-helpers.sh:873: get_num_active_clean: grep -v '^$' .../qa/workunits/ceph-helpers.sh:873: get_num_active_clean: wc -l ../qa/workunits/ceph-helpers.sh:954: is_clean: test 0 = 4 ../qa/workunits/ceph-helpers.sh:954: is_clean: return 1 ../qa/workunits/ceph-helpers.sh:986: wait_for_clean: get_is_making_recovery_progress .../qa/workunits/ceph-helpers.sh:839: get_is_making_recovery_progress: ceph --format xml status .../qa/workunits/ceph-helpers.sh:839: get_is_making_recovery_progress: xmlstarlet sel -t -m //pgmap/recovering_keys_per_sec -v . -o ' ' -t -m //pgmap/recovering_bytes_per_sec -v . -o ' ' -t -m //pgmap/recovering_objects_per_sec -v . ../qa/workunits/ceph-helpers.sh:839: get_is_making_recovery_progress: local progress= ../qa/workunits/ceph-helpers.sh:840: get_is_making_recovery_progress: test -n '' ../qa/workunits/ceph-helpers.sh:988: wait_for_clean: (( timer >= 120 )) .../qa/workunits/ceph-helpers.sh:993: wait_for_clean: get_num_active_clean .../qa/workunits/ceph-helpers.sh:863: get_num_active_clean: local 'expression=(' .../qa/workunits/ceph-helpers.sh:864: get_num_active_clean: expression+='contains(.,'\''active'\'') and ' .../qa/workunits/ceph-helpers.sh:865: get_num_active_clean: expression+='contains(.,'\''clean'\'') and ' .../qa/workunits/ceph-helpers.sh:866: get_num_active_clean: expression+='not(contains(.,'\''stale'\''))' .../qa/workunits/ceph-helpers.sh:867: get_num_active_clean: expression+=')' .../qa/workunits/ceph-helpers.sh:871: get_num_active_clean: ceph --format xml pg dump pgs .../qa/workunits/ceph-helpers.sh:872: get_num_active_clean: xmlstarlet sel -t -m '//pg_stat/state[(contains(.,'\''active'\'') and contains(.,'\''clean'\'') and not(contains(.,'\''stale'\'')))]' -v . -n .../qa/workunits/ceph-helpers.sh:873: get_num_active_clean: grep -v '^$' .../qa/workunits/ceph-helpers.sh:873: get_num_active_clean: wc -l ../qa/workunits/ceph-helpers.sh:993: wait_for_clean: cur_active_clean=0 ../qa/workunits/ceph-helpers.sh:994: wait_for_clean: test 0 '!=' 0 ../qa/workunits/ceph-helpers.sh:998: wait_for_clean: sleep 1 .../qa/workunits/ceph-helpers.sh:999: wait_for_clean: expr 119 + 1 ../qa/workunits/ceph-helpers.sh:999: wait_for_clean: timer=120 ../qa/workunits/ceph-helpers.sh:985: wait_for_clean: is_clean .../qa/workunits/ceph-helpers.sh:952: is_clean: get_num_pgs .../qa/workunits/ceph-helpers.sh:898: get_num_pgs: ceph --format xml status .../qa/workunits/ceph-helpers.sh:899: get_num_pgs: xmlstarlet sel -t -m //pgmap/num_pgs -v . ../qa/workunits/ceph-helpers.sh:952: is_clean: num_pgs=4 ../qa/workunits/ceph-helpers.sh:953: is_clean: test 4 '!=' 0 .../qa/workunits/ceph-helpers.sh:954: is_clean: get_num_active_clean .../qa/workunits/ceph-helpers.sh:863: get_num_active_clean: local 'expression=(' .../qa/workunits/ceph-helpers.sh:864: get_num_active_clean: expression+='contains(.,'\''active'\'') and ' .../qa/workunits/ceph-helpers.sh:865: get_num_active_clean: expression+='contains(.,'\''clean'\'') and ' .../qa/workunits/ceph-helpers.sh:866: get_num_active_clean: expression+='not(contains(.,'\''stale'\''))' .../qa/workunits/ceph-helpers.sh:867: get_num_active_clean: expression+=')' .../qa/workunits/ceph-helpers.sh:871: get_num_active_clean: ceph --format xml pg dump pgs .../qa/workunits/ceph-helpers.sh:872: get_num_active_clean: xmlstarlet sel -t -m '//pg_stat/state[(contains(.,'\''active'\'') and contains(.,'\''clean'\'') and not(contains(.,'\''stale'\'')))]' -v . -n .../qa/workunits/ceph-helpers.sh:873: get_num_active_clean: grep -v '^$' .../qa/workunits/ceph-helpers.sh:873: get_num_active_clean: wc -l ../qa/workunits/ceph-helpers.sh:954: is_clean: test 0 = 4 ../qa/workunits/ceph-helpers.sh:954: is_clean: return 1 ../qa/workunits/ceph-helpers.sh:986: wait_for_clean: get_is_making_recovery_progress .../qa/workunits/ceph-helpers.sh:839: get_is_making_recovery_progress: ceph --format xml status .../qa/workunits/ceph-helpers.sh:839: get_is_making_recovery_progress: xmlstarlet sel -t -m //pgmap/recovering_keys_per_sec -v . -o ' ' -t -m //pgmap/recovering_bytes_per_sec -v . -o ' ' -t -m //pgmap/recovering_objects_per_sec -v . ../qa/workunits/ceph-helpers.sh:839: get_is_making_recovery_progress: local progress= ../qa/workunits/ceph-helpers.sh:840: get_is_making_recovery_progress: test -n '' ../qa/workunits/ceph-helpers.sh:988: wait_for_clean: (( timer >= 120 )) ../qa/workunits/ceph-helpers.sh:989: wait_for_clean: ceph report *** DEVELOPER MODE: setting PATH, PYTHONPATH and LD_LIBRARY_PATH *** report 249181440 { "cluster_fingerprint": "f71870a8-b77b-489c-bf68-762d3d294e6f", "version": "9.0.0-2049-g226d52b", "commit": "226d52b853411bda3e28634bef2f6f484bc48020", "timestamp": "2015-07-15 20:04:04.509438", "tag": "", "health": { "health": { "health_services": [ { "mons": [ { "name": "a", "kb_total": 232279476, "kb_used": 203099020, "kb_avail": 29164072, "avail_percent": 12, "last_updated": "2015-07-15 20:03:16.839547", "store_stats": { "bytes_total": 13632473, "bytes_sst": 603, "bytes_log": 13565952, "bytes_misc": 65918, "last_updated": "0.000000" }, "health": "HEALTH_WARN", "health_detail": "low disk space" } ] } ] }, "timechecks": { "epoch": 2, "round": 0, "round_status": "finished" }, "summary": [ { "severity": "HEALTH_WARN", "summary": "4 pgs stuck inactive" }, { "severity": "HEALTH_WARN", "summary": "4 pgs stuck unclean" }, { "severity": "HEALTH_WARN", "summary": "too few PGs per OSD (4 < min 30)" }, { "severity": "HEALTH_WARN", "summary": "1\/1 in osds are down" }, { "severity": "HEALTH_WARN", "summary": "mon.a low disk space" } ], "overall_status": "HEALTH_WARN", "detail": [] }, "monmap_first_committed": 1, "monmap_last_committed": 1, "monmap": { "epoch": 1, "fsid": "3f1b7cce-0c19-4c66-990c-9bbd1a39c039", "modified": "0.000000", "created": "0.000000", "mons": [ { "rank": 0, "name": "a", "addr": "127.0.0.1:7109\/0" } ] }, "quorum": [ 0 ], "osdmap": { "epoch": 8, "fsid": "3f1b7cce-0c19-4c66-990c-9bbd1a39c039", "created": "2015-07-15 19:59:16.864574", "modified": "2015-07-15 19:59:21.946332", "flags": "", "cluster_snapshot": "", "pool_max": 1, "max_osd": 1, "pools": [ { "pool": 1, "pool_name": "rbd", "flags": 1, "flags_names": "hashpspool", "type": 1, "size": 1, "min_size": 1, "crush_ruleset": 0, "object_hash": 2, "pg_num": 4, "pg_placement_num": 4, "crash_replay_interval": 0, "last_change": "3", "last_force_op_resend": "0", "auid": 0, "snap_mode": "selfmanaged", "snap_seq": 0, "snap_epoch": 0, "pool_snaps": [], "removed_snaps": "[]", "quota_max_bytes": 0, "quota_max_objects": 0, "tiers": [], "tier_of": -1, "read_tier": -1, "write_tier": -1, "cache_mode": "none", "target_max_bytes": 0, "target_max_objects": 0, "cache_target_dirty_ratio_micro": 400000, "cache_target_dirty_high_ratio_micro": 600000, "cache_target_full_ratio_micro": 800000, "cache_min_flush_age": 0, "cache_min_evict_age": 0, "erasure_code_profile": "", "hit_set_params": { "type": "none" }, "hit_set_period": 0, "hit_set_count": 0, "min_read_recency_for_promote": 0, "stripe_width": 0, "expected_num_objects": 0 } ], "osds": [ { "osd": 0, "uuid": "d0bf3a6f-f6c7-47d6-be01-3b66da6c1aed", "up": 0, "in": 1, "weight": 1.000000, "primary_affinity": 1.000000, "last_clean_begin": 0, "last_clean_end": 0, "up_from": 5, "up_thru": 6, "down_at": 8, "lost_at": 0, "public_addr": "127.0.0.1:6804\/29933", "cluster_addr": "127.0.0.1:6805\/29933", "heartbeat_back_addr": "127.0.0.1:6806\/29933", "heartbeat_front_addr": "127.0.0.1:6807\/29933", "state": [ "exists" ] } ], "osd_xinfo": [ { "osd": 0, "down_stamp": "2015-07-15 19:59:21.946332", "laggy_probability": 0.000000, "laggy_interval": 0, "features": 2251799813685247, "old_weight": 0 } ], "pg_temp": [], "primary_temp": [], "blacklist": {}, "erasure_code_profiles": { "default": { "directory": ".libs", "k": "2", "m": "1", "plugin": "jerasure", "technique": "reed_sol_van" } } }, "osd_metadata": [ { "id": 0, "arch": "x86_64", "back_addr": "127.0.0.1:6805\/29933", "backend_filestore_dev_node": "unknown", "backend_filestore_partition_path": "unknown", "ceph_version": "ceph version 9.0.0-2049-g226d52b (226d52b853411bda3e28634bef2f6f484bc48020)", "cpu": "Intel(R) Xeon(R) CPU E5-2620 v2 @ 2.10GHz", "distro": "CentOS", "distro_codename": "Core", "distro_description": "CentOS Linux release 7.0.1406 (Core) ", "distro_version": "7.0.1406", "filestore_backend": "generic", "filestore_f_type": "0xef53", "front_addr": "127.0.0.1:6804\/29933", "hb_back_addr": "127.0.0.1:6806\/29933", "hb_front_addr": "127.0.0.1:6807\/29933", "hostname": "3933a54df967", "kernel_description": "#1 SMP Fri Nov 21 23:26:41 UTC 2014", "kernel_version": "3.17.4-200.fc20.x86_64", "mem_swap_kb": "0", "mem_total_kb": "57683664", "os": "Linux", "osd_data": "testdir\/ceph-helpers\/0", "osd_journal": "testdir\/ceph-helpers\/0\/journal", "osd_objectstore": "filestore" } ], "osdmap_first_committed": 1, "osdmap_last_committed": 8, "crushmap": { "devices": [ { "id": 0, "name": "osd.0" } ], "types": [ { "type_id": 0, "name": "osd" }, { "type_id": 1, "name": "host" }, { "type_id": 2, "name": "chassis" }, { "type_id": 3, "name": "rack" }, { "type_id": 4, "name": "row" }, { "type_id": 5, "name": "pdu" }, { "type_id": 6, "name": "pod" }, { "type_id": 7, "name": "room" }, { "type_id": 8, "name": "datacenter" }, { "type_id": 9, "name": "region" }, { "type_id": 10, "name": "root" } ], "buckets": [ { "id": -1, "name": "default", "type_id": 10, "type_name": "root", "weight": 65536, "alg": "straw", "hash": "rjenkins1", "items": [ { "id": -2, "weight": 65536, "pos": 0 } ] }, { "id": -2, "name": "localhost", "type_id": 1, "type_name": "host", "weight": 65536, "alg": "straw", "hash": "rjenkins1", "items": [ { "id": 0, "weight": 65536, "pos": 0 } ] } ], "rules": [ { "rule_id": 0, "rule_name": "replicated_ruleset", "ruleset": 0, "type": 1, "min_size": 1, "max_size": 10, "steps": [ { "op": "take", "item": -1, "item_name": "default" }, { "op": "choose_firstn", "num": 0, "type": "osd" }, { "op": "emit" } ] } ], "tunables": { "choose_local_tries": 0, "choose_local_fallback_tries": 0, "choose_total_tries": 50, "chooseleaf_descend_once": 1, "chooseleaf_vary_r": 0, "straw_calc_version": 1, "allowed_bucket_algs": 22, "profile": "unknown", "optimal_tunables": 0, "legacy_tunables": 0, "require_feature_tunables": 1, "require_feature_tunables2": 1, "require_feature_tunables3": 0, "has_v2_rules": 0, "has_v3_rules": 0, "has_v4_buckets": 0 } }, "mdsmap": { "epoch": 1, "flags": 0, "created": "0.000000", "modified": "2015-07-15 19:59:16.864159", "tableserver": 0, "root": 0, "session_timeout": 0, "session_autoclose": 0, "max_file_size": 0, "last_failure": 0, "last_failure_osd_epoch": 0, "compat": { "compat": {}, "ro_compat": {}, "incompat": {} }, "max_mds": 0, "in": [], "up": {}, "failed": [], "damaged": [], "stopped": [], "info": {}, "data_pools": [], "metadata_pool": 0, "enabled": false, "fs_name": "cephfs" }, "mdsmap_first_committed": 1, "mdsmap_last_committed": 1, "pgmap": { "version": 9, "stamp": "2015-07-15 19:59:21.949132", "last_osdmap_epoch": 8, "last_pg_scan": 3, "full_ratio": 0.990000, "near_full_ratio": 0.850000, "pg_stats_sum": { "stat_sum": { "num_bytes": 0, "num_objects": 0, "num_object_clones": 0, "num_object_copies": 0, "num_objects_missing_on_primary": 0, "num_objects_degraded": 0, "num_objects_misplaced": 0, "num_objects_unfound": 0, "num_objects_dirty": 0, "num_whiteouts": 0, "num_read": 0, "num_read_kb": 0, "num_write": 0, "num_write_kb": 0, "num_scrub_errors": 0, "num_shallow_scrub_errors": 0, "num_deep_scrub_errors": 0, "num_objects_recovered": 0, "num_bytes_recovered": 0, "num_keys_recovered": 0, "num_objects_omap": 0, "num_objects_hit_set_archive": 0, "num_bytes_hit_set_archive": 0 }, "log_size": 0, "ondisk_log_size": 0, "up": 0, "acting": 0 }, "osd_stats_sum": { "kb": 0, "kb_used": 0, "kb_avail": 0, "hb_in": [], "hb_out": [], "snap_trim_queue_len": 0, "num_snap_trimming": 0, "op_queue_age_hist": { "histogram": [], "upper_bound": 1 }, "fs_perf_stat": { "commit_latency_ms": 0, "apply_latency_ms": 0 } }, "pg_stats_delta": { "stat_sum": { "num_bytes": 0, "num_objects": 0, "num_object_clones": 0, "num_object_copies": 0, "num_objects_missing_on_primary": 0, "num_objects_degraded": 0, "num_objects_misplaced": 0, "num_objects_unfound": 0, "num_objects_dirty": 0, "num_whiteouts": 0, "num_read": 0, "num_read_kb": 0, "num_write": 0, "num_write_kb": 0, "num_scrub_errors": 0, "num_shallow_scrub_errors": 0, "num_deep_scrub_errors": 0, "num_objects_recovered": 0, "num_bytes_recovered": 0, "num_keys_recovered": 0, "num_objects_omap": 0, "num_objects_hit_set_archive": 0, "num_bytes_hit_set_archive": 0 }, "log_size": 0, "ondisk_log_size": 0, "up": 0, "acting": 0 }, "pg_stats": [ { "pgid": "1.2", "version": "0'0", "reported_seq": "0", "reported_epoch": "0", "state": "creating", "last_fresh": "0.000000", "last_change": "0.000000", "last_active": "0.000000", "last_peered": "0.000000", "last_clean": "0.000000", "last_became_active": "0.000000", "last_became_peered": "0.000000", "last_unstale": "0.000000", "last_undegraded": "0.000000", "last_fullsized": "0.000000", "mapping_epoch": 0, "log_start": "0'0", "ondisk_log_start": "0'0", "created": 3, "last_epoch_clean": 0, "parent": "0.0", "parent_split_bits": 0, "last_scrub": "0'0", "last_scrub_stamp": "2015-07-15 19:59:17.874137", "last_deep_scrub": "0'0", "last_deep_scrub_stamp": "2015-07-15 19:59:17.874137", "last_clean_scrub_stamp": "2015-07-15 19:59:17.874137", "log_size": 0, "ondisk_log_size": 0, "stats_invalid": "0", "stat_sum": { "num_bytes": 0, "num_objects": 0, "num_object_clones": 0, "num_object_copies": 0, "num_objects_missing_on_primary": 0, "num_objects_degraded": 0, "num_objects_misplaced": 0, "num_objects_unfound": 0, "num_objects_dirty": 0, "num_whiteouts": 0, "num_read": 0, "num_read_kb": 0, "num_write": 0, "num_write_kb": 0, "num_scrub_errors": 0, "num_shallow_scrub_errors": 0, "num_deep_scrub_errors": 0, "num_objects_recovered": 0, "num_bytes_recovered": 0, "num_keys_recovered": 0, "num_objects_omap": 0, "num_objects_hit_set_archive": 0, "num_bytes_hit_set_archive": 0 }, "up": [], "acting": [], "blocked_by": [], "up_primary": -1, "acting_primary": -1 }, { "pgid": "1.3", "version": "0'0", "reported_seq": "0", "reported_epoch": "0", "state": "creating", "last_fresh": "0.000000", "last_change": "0.000000", "last_active": "0.000000", "last_peered": "0.000000", "last_clean": "0.000000", "last_became_active": "0.000000", "last_became_peered": "0.000000", "last_unstale": "0.000000", "last_undegraded": "0.000000", "last_fullsized": "0.000000", "mapping_epoch": 0, "log_start": "0'0", "ondisk_log_start": "0'0", "created": 3, "last_epoch_clean": 0, "parent": "0.0", "parent_split_bits": 0, "last_scrub": "0'0", "last_scrub_stamp": "2015-07-15 19:59:17.874144", "last_deep_scrub": "0'0", "last_deep_scrub_stamp": "2015-07-15 19:59:17.874144", "last_clean_scrub_stamp": "2015-07-15 19:59:17.874144", "log_size": 0, "ondisk_log_size": 0, "stats_invalid": "0", "stat_sum": { "num_bytes": 0, "num_objects": 0, "num_object_clones": 0, "num_object_copies": 0, "num_objects_missing_on_primary": 0, "num_objects_degraded": 0, "num_objects_misplaced": 0, "num_objects_unfound": 0, "num_objects_dirty": 0, "num_whiteouts": 0, "num_read": 0, "num_read_kb": 0, "num_write": 0, "num_write_kb": 0, "num_scrub_errors": 0, "num_shallow_scrub_errors": 0, "num_deep_scrub_errors": 0, "num_objects_recovered": 0, "num_bytes_recovered": 0, "num_keys_recovered": 0, "num_objects_omap": 0, "num_objects_hit_set_archive": 0, "num_bytes_hit_set_archive": 0 }, "up": [], "acting": [], "blocked_by": [], "up_primary": -1, "acting_primary": -1 }, { "pgid": "1.0", "version": "0'0", "reported_seq": "0", "reported_epoch": "0", "state": "creating", "last_fresh": "0.000000", "last_change": "0.000000", "last_active": "0.000000", "last_peered": "0.000000", "last_clean": "0.000000", "last_became_active": "0.000000", "last_became_peered": "0.000000", "last_unstale": "0.000000", "last_undegraded": "0.000000", "last_fullsized": "0.000000", "mapping_epoch": 0, "log_start": "0'0", "ondisk_log_start": "0'0", "created": 3, "last_epoch_clean": 0, "parent": "0.0", "parent_split_bits": 0, "last_scrub": "0'0", "last_scrub_stamp": "2015-07-15 19:59:17.874125", "last_deep_scrub": "0'0", "last_deep_scrub_stamp": "2015-07-15 19:59:17.874125", "last_clean_scrub_stamp": "2015-07-15 19:59:17.874125", "log_size": 0, "ondisk_log_size": 0, "stats_invalid": "0", "stat_sum": { "num_bytes": 0, "num_objects": 0, "num_object_clones": 0, "num_object_copies": 0, "num_objects_missing_on_primary": 0, "num_objects_degraded": 0, "num_objects_misplaced": 0, "num_objects_unfound": 0, "num_objects_dirty": 0, "num_whiteouts": 0, "num_read": 0, "num_read_kb": 0, "num_write": 0, "num_write_kb": 0, "num_scrub_errors": 0, "num_shallow_scrub_errors": 0, "num_deep_scrub_errors": 0, "num_objects_recovered": 0, "num_bytes_recovered": 0, "num_keys_recovered": 0, "num_objects_omap": 0, "num_objects_hit_set_archive": 0, "num_bytes_hit_set_archive": 0 }, "up": [], "acting": [], "blocked_by": [], "up_primary": -1, "acting_primary": -1 }, { "pgid": "1.1", "version": "0'0", "reported_seq": "0", "reported_epoch": "0", "state": "creating", "last_fresh": "0.000000", "last_change": "0.000000", "last_active": "0.000000", "last_peered": "0.000000", "last_clean": "0.000000", "last_became_active": "0.000000", "last_became_peered": "0.000000", "last_unstale": "0.000000", "last_undegraded": "0.000000", "last_fullsized": "0.000000", "mapping_epoch": 0, "log_start": "0'0", "ondisk_log_start": "0'0", "created": 3, "last_epoch_clean": 0, "parent": "0.0", "parent_split_bits": 0, "last_scrub": "0'0", "last_scrub_stamp": "2015-07-15 19:59:17.874132", "last_deep_scrub": "0'0", "last_deep_scrub_stamp": "2015-07-15 19:59:17.874132", "last_clean_scrub_stamp": "2015-07-15 19:59:17.874132", "log_size": 0, "ondisk_log_size": 0, "stats_invalid": "0", "stat_sum": { "num_bytes": 0, "num_objects": 0, "num_object_clones": 0, "num_object_copies": 0, "num_objects_missing_on_primary": 0, "num_objects_degraded": 0, "num_objects_misplaced": 0, "num_objects_unfound": 0, "num_objects_dirty": 0, "num_whiteouts": 0, "num_read": 0, "num_read_kb": 0, "num_write": 0, "num_write_kb": 0, "num_scrub_errors": 0, "num_shallow_scrub_errors": 0, "num_deep_scrub_errors": 0, "num_objects_recovered": 0, "num_bytes_recovered": 0, "num_keys_recovered": 0, "num_objects_omap": 0, "num_objects_hit_set_archive": 0, "num_bytes_hit_set_archive": 0 }, "up": [], "acting": [], "blocked_by": [], "up_primary": -1, "acting_primary": -1 } ], "pool_stats": [ { "poolid": 1, "stat_sum": { "num_bytes": 0, "num_objects": 0, "num_object_clones": 0, "num_object_copies": 0, "num_objects_missing_on_primary": 0, "num_objects_degraded": 0, "num_objects_misplaced": 0, "num_objects_unfound": 0, "num_objects_dirty": 0, "num_whiteouts": 0, "num_read": 0, "num_read_kb": 0, "num_write": 0, "num_write_kb": 0, "num_scrub_errors": 0, "num_shallow_scrub_errors": 0, "num_deep_scrub_errors": 0, "num_objects_recovered": 0, "num_bytes_recovered": 0, "num_keys_recovered": 0, "num_objects_omap": 0, "num_objects_hit_set_archive": 0, "num_bytes_hit_set_archive": 0 }, "log_size": 0, "ondisk_log_size": 0, "up": 0, "acting": 0 } ], "osd_stats": [] }, "pgmap_first_committed": 1, "pgmap_last_committed": 9, "auth": { "first_committed": 1, "last_committed": 3, "num_secrets": 1 }, "paxos": { "first_committed": 1, "last_committed": 519, "last_pn": 0, "accepted_pn": 0 } } ../qa/workunits/ceph-helpers.sh:990: wait_for_clean: return 1 ../qa/workunits/ceph-helpers.sh:937: test_get_last_scrub_stamp: return 1 ../qa/workunits/ceph-helpers.sh:1199: run_tests: return 1