main: 115: setup testdir/mon_rebuild_store setup: 21: local dir=testdir/mon_rebuild_store setup: 22: teardown testdir/mon_rebuild_store teardown: 27: local dir=testdir/mon_rebuild_store teardown: 28: kill_daemons testdir/mon_rebuild_store kill_daemons: 68: local dir=testdir/mon_rebuild_store kkill_daemons: 67: find testdir/mon_rebuild_store kkill_daemons: 67: grep '\.pid' find: ‘testdir/mon_rebuild_store’: No such file or directory tteardown: 29: stat -f -c %T . teardown: 29: '[' xfs == btrfs ']' teardown: 32: rm -fr testdir/mon_rebuild_store setup: 23: mkdir -p testdir/mon_rebuild_store main: 116: local code main: 117: run testdir/mon_rebuild_store run: 8: local dir=testdir/mon_rebuild_store run: 9: export CEPH_CONF=testdir/mon_rebuild_store/ceph.conf run: 9: CEPH_CONF=testdir/mon_rebuild_store/ceph.conf run: 10: export CEPH_MON=127.0.0.1:7107 run: 10: CEPH_MON=127.0.0.1:7107 run: 11: export CEPH_ARGS run: 12: CEPH_ARGS+='--fsid=21974086-5ea0-4470-a25b-bfc29f7fd72d --auth-supported=none ' run: 13: CEPH_ARGS+='--mon-host=127.0.0.1:7107 ' run: 15: test_mon_rebuild_store testdir/mon_rebuild_store test_mon_rebuild_store: 19: local dir=testdir/mon_rebuild_store test_mon_rebuild_store: 21: run_mon testdir/mon_rebuild_store a --public-addr 127.0.0.1:7107 run_mon: 36: local dir=testdir/mon_rebuild_store run_mon: 37: shift run_mon: 38: local id=a run_mon: 39: shift run_mon: 40: dir+=/a run_mon: 43: ./ceph-mon --id a --mkfs --mon-data=testdir/mon_rebuild_store/a --run-dir=testdir/mon_rebuild_store/a --public-addr 127.0.0.1:7107 2017-02-24 10:13:25.425854 7f68b059b7c0 -1 did not load config file, using default settings. ./ceph-mon: renaming mon.noname-a 127.0.0.1:7107/0 to mon.a ./ceph-mon: set fsid to 21974086-5ea0-4470-a25b-bfc29f7fd72d ./ceph-mon: created monfs at testdir/mon_rebuild_store/a for mon.a run_mon: 49: ./ceph-mon --id a --mon-osd-full-ratio=.99 --mon-data-avail-crit=1 --paxos-propose-interval=0.1 --osd-crush-chooseleaf-type=0 --osd-pool-default-erasure-code-directory=.libs --debug-mon 20 --debug-ms 20 --debug-paxos 20 --chdir= --mon-data=testdir/mon_rebuild_store/a --log-file=testdir/mon_rebuild_store/a/log --mon-cluster-log-file=testdir/mon_rebuild_store/a/log --run-dir=testdir/mon_rebuild_store/a '--pid-file=testdir/mon_rebuild_store/a/$name.pid' --public-addr 127.0.0.1:7107 2017-02-24 10:13:25.453805 7fdb77e587c0 -1 did not load config file, using default settings. test_mon_rebuild_store: 23: cat test_mon_rebuild_store: 34: run_osd testdir/mon_rebuild_store 0 run_osd: 20: local dir=testdir/mon_rebuild_store run_osd: 21: shift run_osd: 22: local id=0 run_osd: 23: shift run_osd: 24: local osd_data=testdir/mon_rebuild_store/0 run_osd: 26: local ceph_disk_args run_osd: 27: ceph_disk_args+=' --statedir=testdir/mon_rebuild_store' run_osd: 28: ceph_disk_args+=' --sysconfdir=testdir/mon_rebuild_store' run_osd: 29: ceph_disk_args+=' --prepend-to-path=' run_osd: 30: ceph_disk_args+=' --verbose' run_osd: 32: touch testdir/mon_rebuild_store/ceph.conf run_osd: 34: mkdir -p testdir/mon_rebuild_store/0 run_osd: 35: ./ceph-disk --statedir=testdir/mon_rebuild_store --sysconfdir=testdir/mon_rebuild_store --prepend-to-path= --verbose prepare testdir/mon_rebuild_store/0 INFO:ceph-disk:Running command: ceph-osd --cluster=ceph --show-config-value=fsid INFO:ceph-disk:Running command: ceph-conf --cluster=ceph --name=osd. --lookup osd_mkfs_type INFO:ceph-disk:Running command: ceph-conf --cluster=ceph --name=osd. --lookup osd_fs_type INFO:ceph-disk:Running command: ceph-conf --cluster=ceph --name=osd. --lookup osd_mkfs_options_xfs INFO:ceph-disk:Running command: ceph-conf --cluster=ceph --name=osd. --lookup osd_fs_mkfs_options_xfs INFO:ceph-disk:Running command: ceph-conf --cluster=ceph --name=osd. --lookup osd_mount_options_xfs INFO:ceph-disk:Running command: ceph-conf --cluster=ceph --name=osd. --lookup osd_fs_mount_options_xfs INFO:ceph-disk:Running command: ceph-osd --cluster=ceph --show-config-value=osd_journal_size INFO:ceph-disk:Running command: ceph-conf --cluster=ceph --name=osd. --lookup osd_cryptsetup_parameters INFO:ceph-disk:Running command: ceph-conf --cluster=ceph --name=osd. --lookup osd_dmcrypt_key_size INFO:ceph-disk:Running command: ceph-conf --cluster=ceph --name=osd. --lookup osd_dmcrypt_type DEBUG:ceph-disk:Preparing osd data dir testdir/mon_rebuild_store/0 run_osd: 38: local 'ceph_args=--fsid=21974086-5ea0-4470-a25b-bfc29f7fd72d --auth-supported=none --mon-host=127.0.0.1:7107 ' run_osd: 39: ceph_args+=' --osd-backfill-full-ratio=.99' run_osd: 40: ceph_args+=' --osd-failsafe-full-ratio=.99' run_osd: 41: ceph_args+=' --osd-journal-size=100' run_osd: 42: ceph_args+=' --osd-data=testdir/mon_rebuild_store/0' run_osd: 43: ceph_args+=' --chdir=' run_osd: 44: ceph_args+=' --osd-pool-default-erasure-code-directory=.libs' run_osd: 45: ceph_args+=' --run-dir=testdir/mon_rebuild_store' run_osd: 46: ceph_args+=' --debug-osd=20' run_osd: 47: ceph_args+=' --debug-filestore=20' run_osd: 48: ceph_args+=' --log-file=testdir/mon_rebuild_store/osd-$id.log' run_osd: 49: ceph_args+=' --pid-file=testdir/mon_rebuild_store/osd-$id.pid' run_osd: 50: ceph_args+=' ' run_osd: 51: ceph_args+= run_osd: 52: mkdir -p testdir/mon_rebuild_store/0 run_osd: 53: CEPH_ARGS='--fsid=21974086-5ea0-4470-a25b-bfc29f7fd72d --auth-supported=none --mon-host=127.0.0.1:7107 --osd-backfill-full-ratio=.99 --osd-failsafe-full-ratio=.99 --osd-journal-size=100 --osd-data=testdir/mon_rebuild_store/0 --chdir= --osd-pool-default-erasure-code-directory=.libs --run-dir=testdir/mon_rebuild_store --debug-osd=20 --debug-filestore=20 --log-file=testdir/mon_rebuild_store/osd-$id.log --pid-file=testdir/mon_rebuild_store/osd-$id.pid ' run_osd: 53: ./ceph-disk --statedir=testdir/mon_rebuild_store --sysconfdir=testdir/mon_rebuild_store --prepend-to-path= --verbose activate --mark-init=none testdir/mon_rebuild_store/0 DEBUG:ceph-disk:Cluster uuid is 21974086-5ea0-4470-a25b-bfc29f7fd72d INFO:ceph-disk:Running command: ceph-osd --cluster=ceph --show-config-value=fsid DEBUG:ceph-disk:Cluster name is ceph DEBUG:ceph-disk:OSD uuid is ea66c84d-c167-44c9-a6f6-e948ed2c6190 DEBUG:ceph-disk:Allocating OSD id... INFO:ceph-disk:Running command: ceph --cluster ceph --name client.bootstrap-osd --keyring testdir/mon_rebuild_store/bootstrap-osd/ceph.keyring osd create --concise ea66c84d-c167-44c9-a6f6-e948ed2c6190 *** DEVELOPER MODE: setting PATH, PYTHONPATH and LD_LIBRARY_PATH *** DEBUG:ceph-disk:OSD id is 0 DEBUG:ceph-disk:Initializing OSD... INFO:ceph-disk:Running command: ceph --cluster ceph --name client.bootstrap-osd --keyring testdir/mon_rebuild_store/bootstrap-osd/ceph.keyring mon getmap -o testdir/mon_rebuild_store/0/activate.monmap *** DEVELOPER MODE: setting PATH, PYTHONPATH and LD_LIBRARY_PATH *** got monmap epoch 1 INFO:ceph-disk:Running command: ceph-osd --cluster ceph --mkfs --mkkey -i 0 --monmap testdir/mon_rebuild_store/0/activate.monmap --osd-data testdir/mon_rebuild_store/0 --osd-journal testdir/mon_rebuild_store/0/journal --osd-uuid ea66c84d-c167-44c9-a6f6-e948ed2c6190 --keyring testdir/mon_rebuild_store/0/keyring 2017-02-24 10:13:27.445954 7fd2a0d3f7c0 -1 journal FileJournal::_open: disabling aio for non-block journal. Use journal_force_aio to force use of aio anyway 2017-02-24 10:13:27.495484 7fd2a0d3f7c0 -1 journal FileJournal::_open: disabling aio for non-block journal. Use journal_force_aio to force use of aio anyway 2017-02-24 10:13:27.496771 7fd2a0d3f7c0 -1 filestore(testdir/mon_rebuild_store/0) could not find -1/23c2fcde/osd_superblock/0 in index: (2) No such file or directory 2017-02-24 10:13:27.544257 7fd2a0d3f7c0 -1 created object store testdir/mon_rebuild_store/0 journal testdir/mon_rebuild_store/0/journal for osd.0 fsid 21974086-5ea0-4470-a25b-bfc29f7fd72d 2017-02-24 10:13:27.544337 7fd2a0d3f7c0 -1 auth: error reading file: testdir/mon_rebuild_store/0/keyring: can't open testdir/mon_rebuild_store/0/keyring: (2) No such file or directory 2017-02-24 10:13:27.544581 7fd2a0d3f7c0 -1 created new key in keyring testdir/mon_rebuild_store/0/keyring DEBUG:ceph-disk:Authorizing OSD key... INFO:ceph-disk:Running command: ceph --cluster ceph --name client.bootstrap-osd --keyring testdir/mon_rebuild_store/bootstrap-osd/ceph.keyring auth add osd.0 -i testdir/mon_rebuild_store/0/keyring osd allow * mon allow profile osd *** DEVELOPER MODE: setting PATH, PYTHONPATH and LD_LIBRARY_PATH *** added key for osd.0 DEBUG:ceph-disk:ceph osd.0 data dir is ready at testdir/mon_rebuild_store/0 INFO:ceph-disk:Running command: ceph-osd --cluster=ceph --id=0 --osd-data=testdir/mon_rebuild_store/0 --osd-journal=testdir/mon_rebuild_store/0/journal starting osd.0 at :/0 osd_data testdir/mon_rebuild_store/0 testdir/mon_rebuild_store/0/journal rrun_osd: 58: cat testdir/mon_rebuild_store/0/whoami run_osd: 58: '[' 0 = 0 ']' run_osd: 60: ./ceph osd crush create-or-move 0 1 root=default host=localhost *** DEVELOPER MODE: setting PATH, PYTHONPATH and LD_LIBRARY_PATH *** create-or-move updating item name 'osd.0' weight 1 at location {host=localhost,root=default} to crush map run_osd: 62: status=1 run_osd: 63: (( i=0 )) run_osd: 63: (( i < 60 )) run_osd: 64: ./ceph osd dump run_osd: 64: grep 'osd.0 up' *** DEVELOPER MODE: setting PATH, PYTHONPATH and LD_LIBRARY_PATH *** osd.0 up in weight 1 up_from 3 up_thru 4 down_at 0 last_clean_interval [0,0) 127.0.0.1:6800/52287 127.0.0.1:6801/52287 127.0.0.1:6802/52287 127.0.0.1:6803/52287 exists,up ea66c84d-c167-44c9-a6f6-e948ed2c6190 run_osd: 67: status=0 run_osd: 68: break run_osd: 72: return 0 test_mon_rebuild_store: 35: run_osd testdir/mon_rebuild_store 1 run_osd: 20: local dir=testdir/mon_rebuild_store run_osd: 21: shift run_osd: 22: local id=1 run_osd: 23: shift run_osd: 24: local osd_data=testdir/mon_rebuild_store/1 run_osd: 26: local ceph_disk_args run_osd: 27: ceph_disk_args+=' --statedir=testdir/mon_rebuild_store' run_osd: 28: ceph_disk_args+=' --sysconfdir=testdir/mon_rebuild_store' run_osd: 29: ceph_disk_args+=' --prepend-to-path=' run_osd: 30: ceph_disk_args+=' --verbose' run_osd: 32: touch testdir/mon_rebuild_store/ceph.conf run_osd: 34: mkdir -p testdir/mon_rebuild_store/1 run_osd: 35: ./ceph-disk --statedir=testdir/mon_rebuild_store --sysconfdir=testdir/mon_rebuild_store --prepend-to-path= --verbose prepare testdir/mon_rebuild_store/1 INFO:ceph-disk:Running command: ceph-osd --cluster=ceph --show-config-value=fsid INFO:ceph-disk:Running command: ceph-conf --cluster=ceph --name=osd. --lookup osd_mkfs_type INFO:ceph-disk:Running command: ceph-conf --cluster=ceph --name=osd. --lookup osd_fs_type INFO:ceph-disk:Running command: ceph-conf --cluster=ceph --name=osd. --lookup osd_mkfs_options_xfs INFO:ceph-disk:Running command: ceph-conf --cluster=ceph --name=osd. --lookup osd_fs_mkfs_options_xfs INFO:ceph-disk:Running command: ceph-conf --cluster=ceph --name=osd. --lookup osd_mount_options_xfs INFO:ceph-disk:Running command: ceph-conf --cluster=ceph --name=osd. --lookup osd_fs_mount_options_xfs INFO:ceph-disk:Running command: ceph-osd --cluster=ceph --show-config-value=osd_journal_size INFO:ceph-disk:Running command: ceph-conf --cluster=ceph --name=osd. --lookup osd_cryptsetup_parameters INFO:ceph-disk:Running command: ceph-conf --cluster=ceph --name=osd. --lookup osd_dmcrypt_key_size INFO:ceph-disk:Running command: ceph-conf --cluster=ceph --name=osd. --lookup osd_dmcrypt_type DEBUG:ceph-disk:Preparing osd data dir testdir/mon_rebuild_store/1 run_osd: 38: local 'ceph_args=--fsid=21974086-5ea0-4470-a25b-bfc29f7fd72d --auth-supported=none --mon-host=127.0.0.1:7107 ' run_osd: 39: ceph_args+=' --osd-backfill-full-ratio=.99' run_osd: 40: ceph_args+=' --osd-failsafe-full-ratio=.99' run_osd: 41: ceph_args+=' --osd-journal-size=100' run_osd: 42: ceph_args+=' --osd-data=testdir/mon_rebuild_store/1' run_osd: 43: ceph_args+=' --chdir=' run_osd: 44: ceph_args+=' --osd-pool-default-erasure-code-directory=.libs' run_osd: 45: ceph_args+=' --run-dir=testdir/mon_rebuild_store' run_osd: 46: ceph_args+=' --debug-osd=20' run_osd: 47: ceph_args+=' --debug-filestore=20' run_osd: 48: ceph_args+=' --log-file=testdir/mon_rebuild_store/osd-$id.log' run_osd: 49: ceph_args+=' --pid-file=testdir/mon_rebuild_store/osd-$id.pid' run_osd: 50: ceph_args+=' ' run_osd: 51: ceph_args+= run_osd: 52: mkdir -p testdir/mon_rebuild_store/1 run_osd: 53: CEPH_ARGS='--fsid=21974086-5ea0-4470-a25b-bfc29f7fd72d --auth-supported=none --mon-host=127.0.0.1:7107 --osd-backfill-full-ratio=.99 --osd-failsafe-full-ratio=.99 --osd-journal-size=100 --osd-data=testdir/mon_rebuild_store/1 --chdir= --osd-pool-default-erasure-code-directory=.libs --run-dir=testdir/mon_rebuild_store --debug-osd=20 --debug-filestore=20 --log-file=testdir/mon_rebuild_store/osd-$id.log --pid-file=testdir/mon_rebuild_store/osd-$id.pid ' run_osd: 53: ./ceph-disk --statedir=testdir/mon_rebuild_store --sysconfdir=testdir/mon_rebuild_store --prepend-to-path= --verbose activate --mark-init=none testdir/mon_rebuild_store/1 DEBUG:ceph-disk:Cluster uuid is 21974086-5ea0-4470-a25b-bfc29f7fd72d INFO:ceph-disk:Running command: ceph-osd --cluster=ceph --show-config-value=fsid DEBUG:ceph-disk:Cluster name is ceph DEBUG:ceph-disk:OSD uuid is f9772a92-d2e5-4d46-bbee-3187ac9dd66b DEBUG:ceph-disk:Allocating OSD id... INFO:ceph-disk:Running command: ceph --cluster ceph --name client.bootstrap-osd --keyring testdir/mon_rebuild_store/bootstrap-osd/ceph.keyring osd create --concise f9772a92-d2e5-4d46-bbee-3187ac9dd66b *** DEVELOPER MODE: setting PATH, PYTHONPATH and LD_LIBRARY_PATH *** DEBUG:ceph-disk:OSD id is 1 DEBUG:ceph-disk:Initializing OSD... INFO:ceph-disk:Running command: ceph --cluster ceph --name client.bootstrap-osd --keyring testdir/mon_rebuild_store/bootstrap-osd/ceph.keyring mon getmap -o testdir/mon_rebuild_store/1/activate.monmap *** DEVELOPER MODE: setting PATH, PYTHONPATH and LD_LIBRARY_PATH *** got monmap epoch 1 INFO:ceph-disk:Running command: ceph-osd --cluster ceph --mkfs --mkkey -i 1 --monmap testdir/mon_rebuild_store/1/activate.monmap --osd-data testdir/mon_rebuild_store/1 --osd-journal testdir/mon_rebuild_store/1/journal --osd-uuid f9772a92-d2e5-4d46-bbee-3187ac9dd66b --keyring testdir/mon_rebuild_store/1/keyring 2017-02-24 10:13:31.591747 7f59804eb7c0 -1 journal FileJournal::_open: disabling aio for non-block journal. Use journal_force_aio to force use of aio anyway 2017-02-24 10:13:31.643259 7f59804eb7c0 -1 journal FileJournal::_open: disabling aio for non-block journal. Use journal_force_aio to force use of aio anyway 2017-02-24 10:13:31.644347 7f59804eb7c0 -1 filestore(testdir/mon_rebuild_store/1) could not find -1/23c2fcde/osd_superblock/0 in index: (2) No such file or directory 2017-02-24 10:13:31.681948 7f59804eb7c0 -1 created object store testdir/mon_rebuild_store/1 journal testdir/mon_rebuild_store/1/journal for osd.1 fsid 21974086-5ea0-4470-a25b-bfc29f7fd72d 2017-02-24 10:13:31.682024 7f59804eb7c0 -1 auth: error reading file: testdir/mon_rebuild_store/1/keyring: can't open testdir/mon_rebuild_store/1/keyring: (2) No such file or directory 2017-02-24 10:13:31.682308 7f59804eb7c0 -1 created new key in keyring testdir/mon_rebuild_store/1/keyring DEBUG:ceph-disk:Authorizing OSD key... INFO:ceph-disk:Running command: ceph --cluster ceph --name client.bootstrap-osd --keyring testdir/mon_rebuild_store/bootstrap-osd/ceph.keyring auth add osd.1 -i testdir/mon_rebuild_store/1/keyring osd allow * mon allow profile osd *** DEVELOPER MODE: setting PATH, PYTHONPATH and LD_LIBRARY_PATH *** added key for osd.1 DEBUG:ceph-disk:ceph osd.1 data dir is ready at testdir/mon_rebuild_store/1 INFO:ceph-disk:Running command: ceph-osd --cluster=ceph --id=1 --osd-data=testdir/mon_rebuild_store/1 --osd-journal=testdir/mon_rebuild_store/1/journal starting osd.1 at :/0 osd_data testdir/mon_rebuild_store/1 testdir/mon_rebuild_store/1/journal rrun_osd: 58: cat testdir/mon_rebuild_store/1/whoami run_osd: 58: '[' 1 = 1 ']' run_osd: 60: ./ceph osd crush create-or-move 1 1 root=default host=localhost *** DEVELOPER MODE: setting PATH, PYTHONPATH and LD_LIBRARY_PATH *** create-or-move updating item name 'osd.1' weight 1 at location {host=localhost,root=default} to crush map run_osd: 62: status=1 run_osd: 63: (( i=0 )) run_osd: 63: (( i < 60 )) run_osd: 64: ./ceph osd dump run_osd: 64: grep 'osd.1 up' *** DEVELOPER MODE: setting PATH, PYTHONPATH and LD_LIBRARY_PATH *** osd.1 up in weight 1 up_from 7 up_thru 8 down_at 0 last_clean_interval [0,0) 127.0.0.1:6804/52686 127.0.0.1:6805/52686 127.0.0.1:6806/52686 127.0.0.1:6807/52686 exists,up f9772a92-d2e5-4d46-bbee-3187ac9dd66b run_osd: 67: status=0 run_osd: 68: break run_osd: 72: return 0 test_mon_rebuild_store: 37: ./rados -p rbd bench 10 write --no-cleanup Maintaining 16 concurrent writes of 4194304 bytes for up to 10 seconds or 0 objects Object prefix: benchmark_data_ol7_52916 sec Cur ops started finished avg MB/s cur MB/s last lat(s) avg lat(s) 0 0 0 0 0 0 - 0 1 16 29 13 51.9849 52 0.394288 0.2862 2 16 38 22 43.9888 36 1.2568 0.621924 3 16 51 35 46.655 52 1.30843 1.06104 4 16 51 35 34.9917 0 - 1.06104 5 16 51 35 27.9934 0 - 1.06104 6 16 51 35 23.3278 0 - 1.06104 7 16 51 35 19.9952 0 - 1.06104 8 16 61 45 22.4946 8 5.44276 2.11098 9 16 66 50 22.2169 20 5.83254 2.47207 10 16 80 64 25.5939 56 1.31547 2.3118 11 16 81 65 23.6308 4 0.957136 2.29096 12 16 81 65 21.6615 0 - 2.29096 13 16 81 65 19.9953 0 - 2.29096 14 16 81 65 18.5672 0 - 2.29096 15 16 81 65 17.3294 0 - 2.29096 Total time run: 15.522862 Total writes made: 81 Write size: 4194304 Bandwidth (MB/sec): 20.8724 Stddev Bandwidth: 21.7117 Max bandwidth (MB/sec): 56 Min bandwidth (MB/sec): 0 Average IOPS: 5 Average Latency(s): 3.05681 Stddev Latency(s): 2.45207 Max latency(s): 7.27272 Min latency(s): 0.19069 test_mon_rebuild_store: 39: kill_daemons testdir/mon_rebuild_store kill_daemons: 68: local dir=testdir/mon_rebuild_store kkill_daemons: 67: find testdir/mon_rebuild_store kkill_daemons: 67: grep '\.pid' kill_daemons: 69: for pidfile in '$(find $dir | grep '\''\.pid'\'')' kkill_daemons: 70: cat testdir/mon_rebuild_store/a/mon.a.pid kill_daemons: 70: pid=52086 kill_daemons: 71: signal=9 kill_daemons: 72: for try in 0 1 1 1 2 3 kill_daemons: 73: kill -9 52086 kill_daemons: 74: signal=0 kill_daemons: 75: sleep 0 kill_daemons: 72: for try in 0 1 1 1 2 3 kill_daemons: 73: kill -0 52086 kill_daemons: 74: signal=0 kill_daemons: 75: sleep 1 kill_daemons: 72: for try in 0 1 1 1 2 3 kill_daemons: 73: kill -0 52086 kill_daemons: 73: break kill_daemons: 69: for pidfile in '$(find $dir | grep '\''\.pid'\'')' kkill_daemons: 70: cat testdir/mon_rebuild_store/osd-0.pid kill_daemons: 70: pid=52289 kill_daemons: 71: signal=9 kill_daemons: 72: for try in 0 1 1 1 2 3 kill_daemons: 73: kill -9 52289 kill_daemons: 74: signal=0 kill_daemons: 75: sleep 0 kill_daemons: 72: for try in 0 1 1 1 2 3 kill_daemons: 73: kill -0 52289 kill_daemons: 74: signal=0 kill_daemons: 75: sleep 1 kill_daemons: 72: for try in 0 1 1 1 2 3 kill_daemons: 73: kill -0 52289 kill_daemons: 73: break kill_daemons: 69: for pidfile in '$(find $dir | grep '\''\.pid'\'')' kkill_daemons: 70: cat testdir/mon_rebuild_store/osd-1.pid kill_daemons: 70: pid=52688 kill_daemons: 71: signal=9 kill_daemons: 72: for try in 0 1 1 1 2 3 kill_daemons: 73: kill -9 52688 kill_daemons: 74: signal=0 kill_daemons: 75: sleep 0 kill_daemons: 72: for try in 0 1 1 1 2 3 kill_daemons: 73: kill -0 52688 kill_daemons: 74: signal=0 kill_daemons: 75: sleep 1 kill_daemons: 72: for try in 0 1 1 1 2 3 kill_daemons: 73: kill -0 52688 kill_daemons: 73: break test_mon_rebuild_store: 41: local ms=testdir/mon_rebuild_store/mon-store test_mon_rebuild_store: 42: mkdir testdir/mon_rebuild_store/mon-store test_mon_rebuild_store: 45: for osd in '{0..1}' test_mon_rebuild_store: 46: ./ceph-objectstore-tool --data-path testdir/mon_rebuild_store/0 --journal-path testdir/mon_rebuild_store/0/journal --op update-mon-db --mon-store-path testdir/mon_rebuild_store/mon-store osd.0 : 0 osdmaps trimmed, 9 osdmaps added. 64 pgs added. test_mon_rebuild_store: 45: for osd in '{0..1}' test_mon_rebuild_store: 46: ./ceph-objectstore-tool --data-path testdir/mon_rebuild_store/1 --journal-path testdir/mon_rebuild_store/1/journal --op update-mon-db --mon-store-path testdir/mon_rebuild_store/mon-store osd.1 : 0 osdmaps trimmed, 0 osdmaps added. 8 pgs added. test_mon_rebuild_store: 49: ./ceph-monstore-tool testdir/mon_rebuild_store/mon-store rebuild terminate called after throwing an instance of 'std::logic_error' what(): basic_string::_S_construct null not valid *** Caught signal (Aborted) ** in thread 7fd433830780 ceph version 0.94.10 (b1e0532418e4631af01acbc0cedd426f1905f4af) 1: ./ceph-monstore-tool() [0x52c204] 2: (()+0xf100) [0x7fd432dac100] 3: (gsignal()+0x37) [0x7fd43144d5f7] 4: (abort()+0x148) [0x7fd43144ece8] 5: (__gnu_cxx::__verbose_terminate_handler()+0x165) [0x7fd431d4e9d5] 6: (()+0x5e946) [0x7fd431d4c946] 7: (()+0x5e973) [0x7fd431d4c973] 8: (()+0x5eb93) [0x7fd431d4cb93] 9: (std::__throw_logic_error(char const*)+0x77) [0x7fd431da1757] 10: (char* std::string::_S_construct(char const*, char const*, std::allocator const&, std::forward_iterator_tag)+0xa1) [0x7fd431dad5a1] 11: (std::basic_string, std::allocator >::basic_string(char const*, std::allocator const&)+0x38) [0x7fd431dad958] 12: (rebuild_monstore(char const*, std::vector >&, MonitorDBStore&)+0x848) [0x5102d8] 13: (main()+0x3e04) [0x50a614] 14: (__libc_start_main()+0xf5) [0x7fd431439b15] 15: ./ceph-monstore-tool() [0x50be39] 2017-02-24 10:13:54.268339 7fd433830780 -1 *** Caught signal (Aborted) ** in thread 7fd433830780 ceph version 0.94.10 (b1e0532418e4631af01acbc0cedd426f1905f4af) 1: ./ceph-monstore-tool() [0x52c204] 2: (()+0xf100) [0x7fd432dac100] 3: (gsignal()+0x37) [0x7fd43144d5f7] 4: (abort()+0x148) [0x7fd43144ece8] 5: (__gnu_cxx::__verbose_terminate_handler()+0x165) [0x7fd431d4e9d5] 6: (()+0x5e946) [0x7fd431d4c946] 7: (()+0x5e973) [0x7fd431d4c973] 8: (()+0x5eb93) [0x7fd431d4cb93] 9: (std::__throw_logic_error(char const*)+0x77) [0x7fd431da1757] 10: (char* std::string::_S_construct(char const*, char const*, std::allocator const&, std::forward_iterator_tag)+0xa1) [0x7fd431dad5a1] 11: (std::basic_string, std::allocator >::basic_string(char const*, std::allocator const&)+0x38) [0x7fd431dad958] 12: (rebuild_monstore(char const*, std::vector >&, MonitorDBStore&)+0x848) [0x5102d8] 13: (main()+0x3e04) [0x50a614] 14: (__libc_start_main()+0xf5) [0x7fd431439b15] 15: ./ceph-monstore-tool() [0x50be39] NOTE: a copy of the executable, or `objdump -rdS ` is needed to interpret this. --- begin dump of recent events --- -14> 2017-02-24 10:13:54.254883 7fd433830780 5 asok(0x3119ef0) register_command perfcounters_dump hook 0x3119ea0 -13> 2017-02-24 10:13:54.254909 7fd433830780 5 asok(0x3119ef0) register_command 1 hook 0x3119ea0 -12> 2017-02-24 10:13:54.254917 7fd433830780 5 asok(0x3119ef0) register_command perf dump hook 0x3119ea0 -11> 2017-02-24 10:13:54.254926 7fd433830780 5 asok(0x3119ef0) register_command perfcounters_schema hook 0x3119ea0 -10> 2017-02-24 10:13:54.254948 7fd433830780 5 asok(0x3119ef0) register_command 2 hook 0x3119ea0 -9> 2017-02-24 10:13:54.254959 7fd433830780 5 asok(0x3119ef0) register_command perf schema hook 0x3119ea0 -8> 2017-02-24 10:13:54.254965 7fd433830780 5 asok(0x3119ef0) register_command perf reset hook 0x3119ea0 -7> 2017-02-24 10:13:54.254971 7fd433830780 5 asok(0x3119ef0) register_command config show hook 0x3119ea0 -6> 2017-02-24 10:13:54.254977 7fd433830780 5 asok(0x3119ef0) register_command config set hook 0x3119ea0 -5> 2017-02-24 10:13:54.254982 7fd433830780 5 asok(0x3119ef0) register_command config get hook 0x3119ea0 -4> 2017-02-24 10:13:54.254989 7fd433830780 5 asok(0x3119ef0) register_command config diff hook 0x3119ea0 -3> 2017-02-24 10:13:54.255002 7fd433830780 5 asok(0x3119ef0) register_command log flush hook 0x3119ea0 -2> 2017-02-24 10:13:54.255008 7fd433830780 5 asok(0x3119ef0) register_command log dump hook 0x3119ea0 -1> 2017-02-24 10:13:54.255014 7fd433830780 5 asok(0x3119ef0) register_command log reopen hook 0x3119ea0 0> 2017-02-24 10:13:54.268339 7fd433830780 -1 *** Caught signal (Aborted) ** in thread 7fd433830780 ceph version 0.94.10 (b1e0532418e4631af01acbc0cedd426f1905f4af) 1: ./ceph-monstore-tool() [0x52c204] 2: (()+0xf100) [0x7fd432dac100] 3: (gsignal()+0x37) [0x7fd43144d5f7] 4: (abort()+0x148) [0x7fd43144ece8] 5: (__gnu_cxx::__verbose_terminate_handler()+0x165) [0x7fd431d4e9d5] 6: (()+0x5e946) [0x7fd431d4c946] 7: (()+0x5e973) [0x7fd431d4c973] 8: (()+0x5eb93) [0x7fd431d4cb93] 9: (std::__throw_logic_error(char const*)+0x77) [0x7fd431da1757] 10: (char* std::string::_S_construct(char const*, char const*, std::allocator const&, std::forward_iterator_tag)+0xa1) [0x7fd431dad5a1] 11: (std::basic_string, std::allocator >::basic_string(char const*, std::allocator const&)+0x38) [0x7fd431dad958] 12: (rebuild_monstore(char const*, std::vector >&, MonitorDBStore&)+0x848) [0x5102d8] 13: (main()+0x3e04) [0x50a614] 14: (__libc_start_main()+0xf5) [0x7fd431439b15] 15: ./ceph-monstore-tool() [0x50be39] NOTE: a copy of the executable, or `objdump -rdS ` is needed to interpret this. --- logging levels --- 0/ 5 none 0/ 1 lockdep 0/ 1 context 1/ 1 crush 1/ 5 mds 1/ 5 mds_balancer 1/ 5 mds_locker 1/ 5 mds_log 1/ 5 mds_log_expire 1/ 5 mds_migrator 0/ 1 buffer 0/ 1 timer 0/ 1 filer 0/ 1 striper 0/ 1 objecter 0/ 5 rados 0/ 5 rbd 0/ 5 rbd_replay 0/ 5 journaler 0/ 5 objectcacher 0/ 5 client 0/ 5 osd 0/ 5 optracker 0/ 5 objclass 1/ 3 filestore 1/ 3 keyvaluestore 1/ 3 journal 0/ 5 ms 1/ 5 mon 0/10 monc 1/ 5 paxos 0/ 5 tp 1/ 5 auth 1/ 5 crypto 1/ 1 finisher 1/ 5 heartbeatmap 1/ 5 perfcounter 1/ 5 rgw 1/10 civetweb 1/ 5 javaclient 1/ 5 asok 1/ 1 throttle 0/ 0 refs 1/ 5 xio -2/-2 (syslog threshold) 99/99 (stderr threshold) max_recent 500 max_new 1000 log_file --- end dump of recent events --- test/mon/mon-rebuild-store.sh: line 18: 53024 Aborted (core dumped) ./ceph-monstore-tool $dir/mon-store rebuild test_mon_rebuild_store: 49: return 1 run: 15: return 1 main: 120: code=1 main: 122: teardown testdir/mon_rebuild_store teardown: 27: local dir=testdir/mon_rebuild_store teardown: 28: kill_daemons testdir/mon_rebuild_store kill_daemons: 68: local dir=testdir/mon_rebuild_store kkill_daemons: 67: find testdir/mon_rebuild_store kkill_daemons: 67: grep '\.pid' kill_daemons: 69: for pidfile in '$(find $dir | grep '\''\.pid'\'')' kkill_daemons: 70: cat testdir/mon_rebuild_store/a/mon.a.pid kill_daemons: 70: pid=52086 kill_daemons: 71: signal=9 kill_daemons: 72: for try in 0 1 1 1 2 3 kill_daemons: 73: kill -9 52086 kill_daemons: 73: break kill_daemons: 69: for pidfile in '$(find $dir | grep '\''\.pid'\'')' kkill_daemons: 70: cat testdir/mon_rebuild_store/osd-0.pid kill_daemons: 70: pid=52289 kill_daemons: 71: signal=9 kill_daemons: 72: for try in 0 1 1 1 2 3 kill_daemons: 73: kill -9 52289 kill_daemons: 73: break kill_daemons: 69: for pidfile in '$(find $dir | grep '\''\.pid'\'')' kkill_daemons: 70: cat testdir/mon_rebuild_store/osd-1.pid kill_daemons: 70: pid=52688 kill_daemons: 71: signal=9 kill_daemons: 72: for try in 0 1 1 1 2 3 kill_daemons: 73: kill -9 52688 kill_daemons: 73: break tteardown: 29: stat -f -c %T . teardown: 29: '[' xfs == btrfs ']' teardown: 32: rm -fr testdir/mon_rebuild_store main: 123: return 1