CEPH_ARGS="--debug-bluestore 20 --err-to-stderr --log-file out" ceph-bluestore-tool fsck --path /var/lib/ceph/osd/ceph-19/ action fsck 2017-09-22 09:46:37.923100 7f6903e8dd00 10 bluestore(/var/lib/ceph/osd/ceph-19/) set_cache_shards 1 2017-09-22 09:46:37.923542 7f6903e8dd00 10 bluestore(/var/lib/ceph/osd/ceph-19/) set_cache_shards 1 2017-09-22 09:46:37.923549 7f6903e8dd00 1 bluestore(/var/lib/ceph/osd/ceph-19/) fsck (shallow) start 2017-09-22 09:46:37.923635 7f6903e8dd00 1 bdev create path /var/lib/ceph/osd/ceph-19//block type kernel 2017-09-22 09:46:37.923644 7f6903e8dd00 1 bdev(0x7f6905cf0600 /var/lib/ceph/osd/ceph-19//block) open path /var/lib/ceph/osd/ceph-19//block 2017-09-22 09:46:37.924183 7f6903e8dd00 1 bdev(0x7f6905cf0600 /var/lib/ceph/osd/ceph-19//block) open size 4000681095168 (0x3a37b2cf000, 3725 GB) block_size 4096 (4096 B) non-rotational 2017-09-22 09:46:37.924198 7f6903e8dd00 10 bluestore(/var/lib/ceph/osd/ceph-19//block) _read_bdev_label 2017-09-22 09:46:37.924471 7f6903e8dd00 10 bluestore(/var/lib/ceph/osd/ceph-19//block) _read_bdev_label got bdev(osd_uuid 2e58ed60-8469-4804-a5ac-a44137653330 size 0x3a37b2cf000 btime 2017-09-21 14:19:40.669997 desc main) 2017-09-22 09:46:37.924505 7f6903e8dd00 1 bluestore(/var/lib/ceph/osd/ceph-19/) _set_cache_sizes max 0.166667 < ratio 0.99 2017-09-22 09:46:37.924528 7f6903e8dd00 1 bluestore(/var/lib/ceph/osd/ceph-19/) _set_cache_sizes cache_size 3221225472 meta 0.833333 kv 0.166667 data 1.49012e-08 2017-09-22 09:46:37.924603 7f6903e8dd00 10 bluestore(/var/lib/ceph/osd/ceph-19/) _open_db kv_backend = rocksdb 2017-09-22 09:46:37.924614 7f6903e8dd00 10 bluestore(/var/lib/ceph/osd/ceph-19/) _open_db do_bluefs = 1 2017-09-22 09:46:37.924619 7f6903e8dd00 10 bluestore(/var/lib/ceph/osd/ceph-19/) _open_db initializing bluefs 2017-09-22 09:46:37.924639 7f6903e8dd00 1 bdev create path /var/lib/ceph/osd/ceph-19//block.db type kernel 2017-09-22 09:46:37.924643 7f6903e8dd00 1 bdev(0x7f6905cf0400 /var/lib/ceph/osd/ceph-19//block.db) open path /var/lib/ceph/osd/ceph-19//block.db 2017-09-22 09:46:37.924788 7f6903e8dd00 1 bdev(0x7f6905cf0400 /var/lib/ceph/osd/ceph-19//block.db) open size 40802189312 (0x980000000, 38912 MB) block_size 4096 (4096 B) non-rotational 2017-09-22 09:46:37.924797 7f6903e8dd00 1 bluefs add_block_device bdev 1 path /var/lib/ceph/osd/ceph-19//block.db size 38912 MB 2017-09-22 09:46:37.924806 7f6903e8dd00 10 bluestore(/var/lib/ceph/osd/ceph-19//block.db) _read_bdev_label 2017-09-22 09:46:37.924984 7f6903e8dd00 10 bluestore(/var/lib/ceph/osd/ceph-19//block.db) _read_bdev_label got bdev(osd_uuid 2e58ed60-8469-4804-a5ac-a44137653330 size 0x980000000 btime 2017-09-21 14:19:40.670223 desc bluefs db) 2017-09-22 09:46:37.925000 7f6903e8dd00 1 bdev create path /var/lib/ceph/osd/ceph-19//block type kernel 2017-09-22 09:46:37.925003 7f6903e8dd00 1 bdev(0x7f6905cf0200 /var/lib/ceph/osd/ceph-19//block) open path /var/lib/ceph/osd/ceph-19//block 2017-09-22 09:46:37.925127 7f6903e8dd00 1 bdev(0x7f6905cf0200 /var/lib/ceph/osd/ceph-19//block) open size 4000681095168 (0x3a37b2cf000, 3725 GB) block_size 4096 (4096 B) non-rotational 2017-09-22 09:46:37.925133 7f6903e8dd00 1 bluefs add_block_device bdev 2 path /var/lib/ceph/osd/ceph-19//block size 3725 GB 2017-09-22 09:46:37.925148 7f6903e8dd00 1 bdev create path /var/lib/ceph/osd/ceph-19//block.wal type kernel 2017-09-22 09:46:37.925153 7f6903e8dd00 1 bdev(0x7f6905cf1400 /var/lib/ceph/osd/ceph-19//block.wal) open path /var/lib/ceph/osd/ceph-19//block.wal 2017-09-22 09:46:37.925277 7f6903e8dd00 1 bdev(0x7f6905cf1400 /var/lib/ceph/osd/ceph-19//block.wal) open size 6442450944 (0x180000000, 6144 MB) block_size 4096 (4096 B) non-rotational 2017-09-22 09:46:37.925284 7f6903e8dd00 1 bluefs add_block_device bdev 0 path /var/lib/ceph/osd/ceph-19//block.wal size 6144 MB 2017-09-22 09:46:37.925286 7f6903e8dd00 10 bluestore(/var/lib/ceph/osd/ceph-19//block.wal) _read_bdev_label 2017-09-22 09:46:37.925465 7f6903e8dd00 10 bluestore(/var/lib/ceph/osd/ceph-19//block.wal) _read_bdev_label got bdev(osd_uuid 2e58ed60-8469-4804-a5ac-a44137653330 size 0x180000000 btime 2017-09-21 14:19:40.670583 desc bluefs wal) 2017-09-22 09:46:37.925495 7f6903e8dd00 1 bluefs mount 2017-09-22 09:46:37.925670 7f6903e8dd00 10 bitmapalloc:BitMapAllocator instance 140089048682864 size 0x180000000 2017-09-22 09:46:37.925696 7f6903e8dd00 10 bitmapalloc:BitMapAllocator instance 140089048682960 size 0x980000000 2017-09-22 09:46:37.926670 7f6903e8dd00 10 bitmapalloc:BitMapAllocator instance 140089048683008 size 0x3a37b2cf000 2017-09-22 09:46:37.930602 7f6903e8dd00 10 bitmapalloc:init_add_free instance 140089048682864 offset 0x1000 length 0x17ffff000 2017-09-22 09:46:37.930609 7f6903e8dd00 20 bitmapalloc:insert_free instance 140089048682864 off 0x100000 len 0x17ff00000 2017-09-22 09:46:37.930616 7f6903e8dd00 10 bitmapalloc:init_add_free instance 140089048682960 offset 0x2000 length 0x97fffe000 2017-09-22 09:46:37.930618 7f6903e8dd00 20 bitmapalloc:insert_free instance 140089048682960 off 0x100000 len 0x97ff00000 2017-09-22 09:46:37.930625 7f6903e8dd00 10 bitmapalloc:init_add_free instance 140089048683008 offset 0x1bf1c600000 length 0x2542600000 2017-09-22 09:46:37.930626 7f6903e8dd00 20 bitmapalloc:insert_free instance 140089048683008 off 0x1bf1c600000 len 0x2542600000 2017-09-22 09:46:37.941592 7f6903e8dd00 10 bitmapalloc:init_rm_free instance 140089048683008 offset 0x1bf29400000 length 0x4100000 2017-09-22 09:46:37.941600 7f6903e8dd00 10 bitmapalloc:init_rm_free instance 140089048682960 offset 0xb00000 length 0x100000 2017-09-22 09:46:37.941602 7f6903e8dd00 10 bitmapalloc:init_rm_free instance 140089048682960 offset 0x2c00000 length 0x100000 2017-09-22 09:46:37.941603 7f6903e8dd00 10 bitmapalloc:init_rm_free instance 140089048682960 offset 0xd00000 length 0x100000 2017-09-22 09:46:37.941604 7f6903e8dd00 10 bitmapalloc:init_rm_free instance 140089048682960 offset 0x100000 length 0x200000 2017-09-22 09:46:37.941605 7f6903e8dd00 10 bitmapalloc:init_rm_free instance 140089048682960 offset 0x400000 length 0x700000 2017-09-22 09:46:37.941606 7f6903e8dd00 10 bitmapalloc:init_rm_free instance 140089048682960 offset 0xc00000 length 0x100000 2017-09-22 09:46:37.941607 7f6903e8dd00 10 bitmapalloc:init_rm_free instance 140089048682960 offset 0xe00000 length 0x1b00000 2017-09-22 09:46:37.941608 7f6903e8dd00 10 bitmapalloc:init_rm_free instance 140089048682960 offset 0x10a200000 length 0xa00000 2017-09-22 09:46:37.941610 7f6903e8dd00 10 bitmapalloc:init_rm_free instance 140089048682960 offset 0x106000000 length 0x4200000 2017-09-22 09:46:37.941611 7f6903e8dd00 10 bitmapalloc:init_rm_free instance 140089048682960 offset 0x101e00000 length 0x4200000 2017-09-22 09:46:37.941612 7f6903e8dd00 10 bitmapalloc:init_rm_free instance 140089048682960 offset 0xf4200000 length 0x500000 2017-09-22 09:46:37.941613 7f6903e8dd00 10 bitmapalloc:init_rm_free instance 140089048682960 offset 0xfe100000 length 0x3d00000 2017-09-22 09:46:37.941615 7f6903e8dd00 10 bitmapalloc:init_rm_free instance 140089048682960 offset 0xf0000000 length 0x4200000 2017-09-22 09:46:37.941618 7f6903e8dd00 10 bitmapalloc:init_rm_free instance 140089048682960 offset 0xeb700000 length 0xe00000 2017-09-22 09:46:37.941619 7f6903e8dd00 10 bitmapalloc:init_rm_free instance 140089048682960 offset 0xecc00000 length 0x3400000 2017-09-22 09:46:37.941620 7f6903e8dd00 10 bitmapalloc:init_rm_free instance 140089048682960 offset 0xb8900000 length 0x500000 2017-09-22 09:46:37.941621 7f6903e8dd00 10 bitmapalloc:init_rm_free instance 140089048682960 offset 0xd9800000 length 0x900000 2017-09-22 09:46:37.941622 7f6903e8dd00 10 bitmapalloc:init_rm_free instance 140089048682960 offset 0xe8300000 length 0x3400000 2017-09-22 09:46:37.941623 7f6903e8dd00 10 bitmapalloc:init_rm_free instance 140089048682960 offset 0xb4700000 length 0x4200000 2017-09-22 09:46:37.941624 7f6903e8dd00 10 bitmapalloc:init_rm_free instance 140089048682960 offset 0xb0500000 length 0x1800000 2017-09-22 09:46:37.941626 7f6903e8dd00 10 bitmapalloc:init_rm_free instance 140089048682960 offset 0xb3800000 length 0xf00000 2017-09-22 09:46:37.941627 7f6903e8dd00 10 bitmapalloc:init_rm_free instance 140089048682960 offset 0xa5600000 length 0x2d00000 2017-09-22 09:46:37.941628 7f6903e8dd00 10 bitmapalloc:init_rm_free instance 140089048682960 offset 0xaae00000 length 0x600000 2017-09-22 09:46:37.941629 7f6903e8dd00 10 bitmapalloc:init_rm_free instance 140089048682960 offset 0xaf600000 length 0xf00000 2017-09-22 09:46:37.941630 7f6903e8dd00 10 bitmapalloc:init_rm_free instance 140089048682960 offset 0xa1400000 length 0x4200000 2017-09-22 09:46:37.941631 7f6903e8dd00 10 bitmapalloc:init_rm_free instance 140089048682960 offset 0x74900000 length 0x900000 2017-09-22 09:46:37.941633 7f6903e8dd00 10 bitmapalloc:init_rm_free instance 140089048682960 offset 0x79100000 length 0x2500000 2017-09-22 09:46:37.941634 7f6903e8dd00 10 bitmapalloc:init_rm_free instance 140089048682960 offset 0x83700000 length 0x1400000 2017-09-22 09:46:37.941635 7f6903e8dd00 10 bitmapalloc:init_rm_free instance 140089048682960 offset 0x70700000 length 0x4200000 2017-09-22 09:46:37.941636 7f6903e8dd00 10 bitmapalloc:init_rm_free instance 140089048682960 offset 0x58500000 length 0x2100000 2017-09-22 09:46:37.941637 7f6903e8dd00 10 bitmapalloc:init_rm_free instance 140089048682960 offset 0x6a400000 length 0x2100000 2017-09-22 09:46:37.941638 7f6903e8dd00 10 bitmapalloc:init_rm_free instance 140089048682960 offset 0x50a00000 length 0x2a00000 2017-09-22 09:46:37.941639 7f6903e8dd00 10 bitmapalloc:init_rm_free instance 140089048682960 offset 0x53500000 length 0x1700000 2017-09-22 09:46:37.941640 7f6903e8dd00 10 bitmapalloc:init_rm_free instance 140089048682960 offset 0x58400000 length 0x100000 2017-09-22 09:46:37.941641 7f6903e8dd00 10 bitmapalloc:init_rm_free instance 140089048682960 offset 0x2a500000 length 0x2500000 2017-09-22 09:46:37.941642 7f6903e8dd00 10 bitmapalloc:init_rm_free instance 140089048682960 offset 0x2fb00000 length 0xa00000 2017-09-22 09:46:37.941643 7f6903e8dd00 10 bitmapalloc:init_rm_free instance 140089048682960 offset 0x34800000 length 0x1100000 2017-09-22 09:46:37.941645 7f6903e8dd00 10 bitmapalloc:init_rm_free instance 140089048682960 offset 0x25e00000 length 0x1e00000 2017-09-22 09:46:37.941646 7f6903e8dd00 10 bitmapalloc:init_rm_free instance 140089048682960 offset 0x28800000 length 0x1d00000 2017-09-22 09:46:37.941646 7f6903e8dd00 10 bitmapalloc:init_rm_free instance 140089048682960 offset 0xc500000 length 0x4200000 2017-09-22 09:46:37.941648 7f6903e8dd00 10 bitmapalloc:init_rm_free instance 140089048682960 offset 0x12d500000 length 0x4200000 2017-09-22 09:46:37.941649 7f6903e8dd00 10 bitmapalloc:init_rm_free instance 140089048682960 offset 0x87d00000 length 0xc00000 2017-09-22 09:46:37.941651 7f6903e8dd00 10 bitmapalloc:init_rm_free instance 140089048682960 offset 0x80500000 length 0x3200000 2017-09-22 09:46:37.941651 7f6903e8dd00 10 bitmapalloc:init_rm_free instance 140089048682960 offset 0x86c00000 length 0x1100000 2017-09-22 09:46:37.941652 7f6903e8dd00 10 bitmapalloc:init_rm_free instance 140089048682960 offset 0x75c00000 length 0x600000 2017-09-22 09:46:37.941653 7f6903e8dd00 10 bitmapalloc:init_rm_free instance 140089048682960 offset 0x7c900000 length 0x3c00000 2017-09-22 09:46:37.941654 7f6903e8dd00 10 bitmapalloc:init_rm_free instance 140089048682960 offset 0x66c00000 length 0x3800000 2017-09-22 09:46:37.941656 7f6903e8dd00 10 bitmapalloc:init_rm_free instance 140089048682960 offset 0x75200000 length 0xa00000 2017-09-22 09:46:37.941656 7f6903e8dd00 10 bitmapalloc:init_rm_free instance 140089048682960 offset 0x62a00000 length 0x4200000 2017-09-22 09:46:37.941657 7f6903e8dd00 10 bitmapalloc:init_rm_free instance 140089048682960 offset 0x32200000 length 0x2600000 2017-09-22 09:46:37.941659 7f6903e8dd00 10 bitmapalloc:init_rm_free instance 140089048682960 offset 0x38a00000 length 0x1c00000 2017-09-22 09:46:37.941660 7f6903e8dd00 10 bitmapalloc:init_rm_free instance 140089048682960 offset 0x2d600000 length 0x2500000 2017-09-22 09:46:37.941662 7f6903e8dd00 10 bitmapalloc:init_rm_free instance 140089048682960 offset 0x30500000 length 0x1d00000 2017-09-22 09:46:37.941663 7f6903e8dd00 10 bitmapalloc:init_rm_free instance 140089048683008 offset 0x1bf74200000 length 0x3300000 2017-09-22 09:46:37.941664 7f6903e8dd00 10 bitmapalloc:init_rm_free instance 140089048683008 offset 0x1bf66400000 length 0x1400000 2017-09-22 09:46:37.941665 7f6903e8dd00 10 bitmapalloc:init_rm_free instance 140089048683008 offset 0x1bf6be00000 length 0x1000000 2017-09-22 09:46:37.941666 7f6903e8dd00 10 bitmapalloc:init_rm_free instance 140089048683008 offset 0x1bf72400000 length 0x1e00000 2017-09-22 09:46:37.941667 7f6903e8dd00 10 bitmapalloc:init_rm_free instance 140089048682960 offset 0x14900000 length 0x100000 2017-09-22 09:46:37.941668 7f6903e8dd00 10 bitmapalloc:init_rm_free instance 140089048682960 offset 0x15700000 length 0x1700000 2017-09-22 09:46:37.941669 7f6903e8dd00 10 bitmapalloc:init_rm_free instance 140089048682960 offset 0x1b000000 length 0x2700000 2017-09-22 09:46:37.941670 7f6903e8dd00 10 bitmapalloc:init_rm_free instance 140089048682960 offset 0x25b00000 length 0x300000 2017-09-22 09:46:37.941671 7f6903e8dd00 10 bitmapalloc:init_rm_free instance 140089048683008 offset 0x1bf62200000 length 0x4200000 2017-09-22 09:46:37.941672 7f6903e8dd00 10 bitmapalloc:init_rm_free instance 140089048683008 offset 0x1bf59b00000 length 0x1300000 2017-09-22 09:46:37.941673 7f6903e8dd00 10 bitmapalloc:init_rm_free instance 140089048683008 offset 0x1bf5f300000 length 0x2f00000 2017-09-22 09:46:37.941674 7f6903e8dd00 10 bitmapalloc:init_rm_free instance 140089048683008 offset 0x1bf34d00000 length 0x2500000 2017-09-22 09:46:37.941675 7f6903e8dd00 10 bitmapalloc:init_rm_free instance 140089048683008 offset 0x1bf3c100000 length 0x1d00000 2017-09-22 09:46:37.941676 7f6903e8dd00 10 bitmapalloc:init_rm_free instance 140089048682960 offset 0x3a600000 length 0x4200000 2017-09-22 09:46:37.941677 7f6903e8dd00 10 bitmapalloc:init_rm_free instance 140089048682960 offset 0x6c500000 length 0x4200000 2017-09-22 09:46:37.941679 7f6903e8dd00 10 bitmapalloc:init_rm_free instance 140089048682960 offset 0xab400000 length 0x4200000 2017-09-22 09:46:37.941680 7f6903e8dd00 10 bitmapalloc:init_rm_free instance 140089048682960 offset 0x4700000 length 0x100000 2017-09-22 09:46:37.941681 7f6903e8dd00 10 bitmapalloc:init_rm_free instance 140089048682864 offset 0x30100000 length 0x8800000 2017-09-22 09:46:37.941682 7f6903e8dd00 10 bitmapalloc:init_rm_free instance 140089048682960 offset 0x84b00000 length 0x2100000 2017-09-22 09:46:37.941684 7f6903e8dd00 10 bitmapalloc:init_rm_free instance 140089048682960 offset 0x8e100000 length 0x500000 2017-09-22 09:46:37.941684 7f6903e8dd00 10 bitmapalloc:init_rm_free instance 140089048682960 offset 0x8f000000 length 0x900000 2017-09-22 09:46:37.941685 7f6903e8dd00 10 bitmapalloc:init_rm_free instance 140089048682960 offset 0xa0100000 length 0x1300000 2017-09-22 09:46:37.941686 7f6903e8dd00 10 bitmapalloc:init_rm_free instance 140089048682960 offset 0x76200000 length 0x2f00000 2017-09-22 09:46:37.941687 7f6903e8dd00 10 bitmapalloc:init_rm_free instance 140089048682960 offset 0x7b600000 length 0x1300000 2017-09-22 09:46:37.941688 7f6903e8dd00 10 bitmapalloc:init_rm_free instance 140089048682960 offset 0x93b00000 length 0x4200000 2017-09-22 09:46:37.941693 7f6903e8dd00 10 bitmapalloc:init_rm_free instance 140089048682960 offset 0x8f900000 length 0x4200000 2017-09-22 09:46:37.941694 7f6903e8dd00 10 bitmapalloc:init_rm_free instance 140089048682960 offset 0x2b00000 length 0x100000 2017-09-22 09:46:37.941701 7f6903e8dd00 10 bitmapalloc:init_rm_free instance 140089048683008 offset 0x1bf4d600000 length 0x4200000 2017-09-22 09:46:37.941702 7f6903e8dd00 10 bitmapalloc:init_rm_free instance 140089048682960 offset 0x42a00000 length 0x4200000 2017-09-22 09:46:37.941703 7f6903e8dd00 10 bitmapalloc:init_rm_free instance 140089048682960 offset 0x35900000 length 0x3100000 2017-09-22 09:46:37.941704 7f6903e8dd00 10 bitmapalloc:init_rm_free instance 140089048682960 offset 0x4d000000 length 0x1100000 2017-09-22 09:46:37.941705 7f6903e8dd00 10 bitmapalloc:init_rm_free instance 140089048682960 offset 0x300000 length 0x100000 2017-09-22 09:46:37.941706 7f6903e8dd00 10 bitmapalloc:init_rm_free instance 140089048682960 offset 0x50100000 length 0x900000 2017-09-22 09:46:37.941708 7f6903e8dd00 10 bitmapalloc:init_rm_free instance 140089048682960 offset 0x53400000 length 0x100000 2017-09-22 09:46:37.941708 7f6903e8dd00 10 bitmapalloc:init_rm_free instance 140089048682960 offset 0x54c00000 length 0x3800000 2017-09-22 09:46:37.941709 7f6903e8dd00 10 bitmapalloc:init_rm_free instance 140089048682960 offset 0x134c00000 length 0x1200000 2017-09-22 09:46:37.941714 7f6903e8dd00 10 bitmapalloc:init_rm_free instance 140089048682960 offset 0xe1e00000 length 0x4200000 2017-09-22 09:46:37.941715 7f6903e8dd00 10 bitmapalloc:init_rm_free instance 140089048683008 offset 0x1bf42000000 length 0x3e00000 2017-09-22 09:46:37.941716 7f6903e8dd00 10 bitmapalloc:init_rm_free instance 140089048683008 offset 0x1bf48400000 length 0x400000 2017-09-22 09:46:37.941717 7f6903e8dd00 10 bitmapalloc:init_rm_free instance 140089048682960 offset 0x3e800000 length 0x4200000 2017-09-22 09:46:37.941718 7f6903e8dd00 10 bitmapalloc:init_rm_free instance 140089048682960 offset 0xda100000 length 0x2400000 2017-09-22 09:46:37.941719 7f6903e8dd00 10 bitmapalloc:init_rm_free instance 140089048682960 offset 0xe0000000 length 0x1e00000 2017-09-22 09:46:37.941720 7f6903e8dd00 10 bitmapalloc:init_rm_free instance 140089048683008 offset 0x1bf3de00000 length 0x4200000 2017-09-22 09:46:37.941721 7f6903e8dd00 10 bitmapalloc:init_rm_free instance 140089048682960 offset 0x46c00000 length 0x4200000 2017-09-22 09:46:37.941722 7f6903e8dd00 10 bitmapalloc:init_rm_free instance 140089048682960 offset 0x5a600000 length 0x4200000 2017-09-22 09:46:37.941723 7f6903e8dd00 10 bitmapalloc:init_rm_free instance 140089048682960 offset 0xcef00000 length 0x2e00000 2017-09-22 09:46:37.941724 7f6903e8dd00 10 bitmapalloc:init_rm_free instance 140089048682960 offset 0xd4100000 length 0x1400000 2017-09-22 09:46:37.941725 7f6903e8dd00 10 bitmapalloc:init_rm_free instance 140089048682960 offset 0x4ae00000 length 0x2200000 2017-09-22 09:46:37.941726 7f6903e8dd00 10 bitmapalloc:init_rm_free instance 140089048682960 offset 0x4e100000 length 0x2000000 2017-09-22 09:46:37.941727 7f6903e8dd00 10 bitmapalloc:init_rm_free instance 140089048683008 offset 0x1bf20c00000 length 0x4600000 2017-09-22 09:46:37.941728 7f6903e8dd00 10 bitmapalloc:init_rm_free instance 140089048683008 offset 0x1bf1c600000 length 0x4600000 2017-09-22 09:46:37.941728 7f6903e8dd00 10 bitmapalloc:init_rm_free instance 140089048682960 offset 0xd5500000 length 0xb00000 2017-09-22 09:46:37.941729 7f6903e8dd00 10 bitmapalloc:init_rm_free instance 140089048682960 offset 0x5e800000 length 0x4200000 2017-09-22 09:46:37.941731 7f6903e8dd00 10 bitmapalloc:init_rm_free instance 140089048683008 offset 0x1bf48800000 length 0x4200000 2017-09-22 09:46:37.941732 7f6903e8dd00 10 bitmapalloc:init_rm_free instance 140089048682960 offset 0xe6000000 length 0x2300000 2017-09-22 09:46:37.941733 7f6903e8dd00 10 bitmapalloc:init_rm_free instance 140089048682960 offset 0xec500000 length 0x700000 2017-09-22 09:46:37.941734 7f6903e8dd00 10 bitmapalloc:init_rm_free instance 140089048683008 offset 0x1bf25200000 length 0x4200000 2017-09-22 09:46:37.941735 7f6903e8dd00 10 bitmapalloc:init_rm_free instance 140089048683008 offset 0x1bf30b00000 length 0x4200000 2017-09-22 09:46:37.941782 7f6903e8dd00 10 bluestore(/var/lib/ceph/osd/ceph-19/) _open_db set rocksdb_db_paths to db,38762079846 db.slow,3800647040409 2017-09-22 09:46:37.941846 7f6903e8dd00 0 set rocksdb option compaction_readahead_size = 2097152 2017-09-22 09:46:37.941854 7f6903e8dd00 0 set rocksdb option compression = kNoCompression 2017-09-22 09:46:37.941859 7f6903e8dd00 0 set rocksdb option max_write_buffer_number = 4 2017-09-22 09:46:37.941861 7f6903e8dd00 0 set rocksdb option min_write_buffer_number_to_merge = 1 2017-09-22 09:46:37.941870 7f6903e8dd00 0 set rocksdb option recycle_log_file_num = 4 2017-09-22 09:46:37.941872 7f6903e8dd00 0 set rocksdb option writable_file_max_buffer_size = 0 2017-09-22 09:46:37.941881 7f6903e8dd00 0 set rocksdb option write_buffer_size = 268435456 2017-09-22 09:46:37.941902 7f6903e8dd00 0 set rocksdb option compaction_readahead_size = 2097152 2017-09-22 09:46:37.941904 7f6903e8dd00 0 set rocksdb option compression = kNoCompression 2017-09-22 09:46:37.941907 7f6903e8dd00 0 set rocksdb option max_write_buffer_number = 4 2017-09-22 09:46:37.941909 7f6903e8dd00 0 set rocksdb option min_write_buffer_number_to_merge = 1 2017-09-22 09:46:37.941911 7f6903e8dd00 0 set rocksdb option recycle_log_file_num = 4 2017-09-22 09:46:37.941912 7f6903e8dd00 0 set rocksdb option writable_file_max_buffer_size = 0 2017-09-22 09:46:37.941915 7f6903e8dd00 0 set rocksdb option write_buffer_size = 268435456 2017-09-22 09:46:37.942042 7f6903e8dd00 4 rocksdb: RocksDB version: 5.4.0 2017-09-22 09:46:37.942047 7f6903e8dd00 4 rocksdb: Git sha rocksdb_build_git_sha:@0@ 2017-09-22 09:46:37.942048 7f6903e8dd00 4 rocksdb: Compile date Sep 21 2017 2017-09-22 09:46:37.942051 7f6903e8dd00 4 rocksdb: DB SUMMARY 2017-09-22 09:46:37.942078 7f6903e8dd00 4 rocksdb: CURRENT file: CURRENT 2017-09-22 09:46:37.942079 7f6903e8dd00 4 rocksdb: IDENTITY file: IDENTITY 2017-09-22 09:46:37.942084 7f6903e8dd00 4 rocksdb: MANIFEST file: MANIFEST-001678 size: 10657 Bytes 2017-09-22 09:46:37.942086 7f6903e8dd00 4 rocksdb: SST files in db dir, Total Num: 49, files: 001173.sst 001235.sst 001236.sst 001296.sst 001297.sst 001298.sst 001424.sst 001425.sst 001469.sst 2017-09-22 09:46:37.942092 7f6903e8dd00 4 rocksdb: SST files in db.slow dir, Total Num: 14, files: 000628.sst 000680.sst 001040.sst 001490.sst 001491.sst 001492.sst 001493.sst 001494.sst 001495.sst 2017-09-22 09:46:37.942099 7f6903e8dd00 4 rocksdb: Write Ahead Log file in db.wal: 001679.log size: 0 ; 2017-09-22 09:46:37.942101 7f6903e8dd00 4 rocksdb: Options.error_if_exists: 0 2017-09-22 09:46:37.942102 7f6903e8dd00 4 rocksdb: Options.create_if_missing: 0 2017-09-22 09:46:37.942103 7f6903e8dd00 4 rocksdb: Options.paranoid_checks: 1 2017-09-22 09:46:37.942104 7f6903e8dd00 4 rocksdb: Options.env: 0x7f6905f784c0 2017-09-22 09:46:37.942106 7f6903e8dd00 4 rocksdb: Options.info_log: 0x7f6905f78da0 2017-09-22 09:46:37.942107 7f6903e8dd00 4 rocksdb: Options.max_open_files: -1 2017-09-22 09:46:37.942107 7f6903e8dd00 4 rocksdb: Options.max_file_opening_threads: 16 2017-09-22 09:46:37.942108 7f6903e8dd00 4 rocksdb: Options.use_fsync: 0 2017-09-22 09:46:37.942109 7f6903e8dd00 4 rocksdb: Options.max_log_file_size: 0 2017-09-22 09:46:37.942110 7f6903e8dd00 4 rocksdb: Options.max_manifest_file_size: 18446744073709551615 2017-09-22 09:46:37.942111 7f6903e8dd00 4 rocksdb: Options.log_file_time_to_roll: 0 2017-09-22 09:46:37.942111 7f6903e8dd00 4 rocksdb: Options.keep_log_file_num: 1000 2017-09-22 09:46:37.942112 7f6903e8dd00 4 rocksdb: Options.recycle_log_file_num: 4 2017-09-22 09:46:37.942113 7f6903e8dd00 4 rocksdb: Options.allow_fallocate: 1 2017-09-22 09:46:37.942113 7f6903e8dd00 4 rocksdb: Options.allow_mmap_reads: 0 2017-09-22 09:46:37.942114 7f6903e8dd00 4 rocksdb: Options.allow_mmap_writes: 0 2017-09-22 09:46:37.942114 7f6903e8dd00 4 rocksdb: Options.use_direct_reads: 0 2017-09-22 09:46:37.942115 7f6903e8dd00 4 rocksdb: Options.use_direct_io_for_flush_and_compaction: 0 2017-09-22 09:46:37.942116 7f6903e8dd00 4 rocksdb: Options.create_missing_column_families: 0 2017-09-22 09:46:37.942117 7f6903e8dd00 4 rocksdb: Options.db_log_dir: 2017-09-22 09:46:37.942117 7f6903e8dd00 4 rocksdb: Options.wal_dir: db.wal 2017-09-22 09:46:37.942118 7f6903e8dd00 4 rocksdb: Options.table_cache_numshardbits: 6 2017-09-22 09:46:37.942119 7f6903e8dd00 4 rocksdb: Options.max_subcompactions: 1 2017-09-22 09:46:37.942119 7f6903e8dd00 4 rocksdb: Options.max_background_flushes: 1 2017-09-22 09:46:37.942120 7f6903e8dd00 4 rocksdb: Options.WAL_ttl_seconds: 0 2017-09-22 09:46:37.942121 7f6903e8dd00 4 rocksdb: Options.WAL_size_limit_MB: 0 2017-09-22 09:46:37.942122 7f6903e8dd00 4 rocksdb: Options.manifest_preallocation_size: 4194304 2017-09-22 09:46:37.942122 7f6903e8dd00 4 rocksdb: Options.is_fd_close_on_exec: 1 2017-09-22 09:46:37.942123 7f6903e8dd00 4 rocksdb: Options.advise_random_on_open: 1 2017-09-22 09:46:37.942124 7f6903e8dd00 4 rocksdb: Options.db_write_buffer_size: 0 2017-09-22 09:46:37.942124 7f6903e8dd00 4 rocksdb: Options.access_hint_on_compaction_start: 1 2017-09-22 09:46:37.942125 7f6903e8dd00 4 rocksdb: Options.new_table_reader_for_compaction_inputs: 1 2017-09-22 09:46:37.942125 7f6903e8dd00 4 rocksdb: Options.compaction_readahead_size: 2097152 2017-09-22 09:46:37.942126 7f6903e8dd00 4 rocksdb: Options.random_access_max_buffer_size: 1048576 2017-09-22 09:46:37.942127 7f6903e8dd00 4 rocksdb: Options.writable_file_max_buffer_size: 0 2017-09-22 09:46:37.942127 7f6903e8dd00 4 rocksdb: Options.use_adaptive_mutex: 0 2017-09-22 09:46:37.942128 7f6903e8dd00 4 rocksdb: Options.rate_limiter: (nil) 2017-09-22 09:46:37.942129 7f6903e8dd00 4 rocksdb: Options.sst_file_manager.rate_bytes_per_sec: 0 2017-09-22 09:46:37.942130 7f6903e8dd00 4 rocksdb: Options.bytes_per_sync: 0 2017-09-22 09:46:37.942131 7f6903e8dd00 4 rocksdb: Options.wal_bytes_per_sync: 0 2017-09-22 09:46:37.942131 7f6903e8dd00 4 rocksdb: Options.wal_recovery_mode: 0 2017-09-22 09:46:37.942132 7f6903e8dd00 4 rocksdb: Options.enable_thread_tracking: 0 2017-09-22 09:46:37.942132 7f6903e8dd00 4 rocksdb: Options.allow_concurrent_memtable_write: 1 2017-09-22 09:46:37.942133 7f6903e8dd00 4 rocksdb: Options.enable_write_thread_adaptive_yield: 1 2017-09-22 09:46:37.942134 7f6903e8dd00 4 rocksdb: Options.write_thread_max_yield_usec: 100 2017-09-22 09:46:37.942134 7f6903e8dd00 4 rocksdb: Options.write_thread_slow_yield_usec: 3 2017-09-22 09:46:37.942135 7f6903e8dd00 4 rocksdb: Options.row_cache: None 2017-09-22 09:46:37.942136 7f6903e8dd00 4 rocksdb: Options.wal_filter: None 2017-09-22 09:46:37.942136 7f6903e8dd00 4 rocksdb: Options.avoid_flush_during_recovery: 0 2017-09-22 09:46:37.942137 7f6903e8dd00 4 rocksdb: Options.base_background_compactions: 1 2017-09-22 09:46:37.942138 7f6903e8dd00 4 rocksdb: Options.max_background_compactions: 1 2017-09-22 09:46:37.942139 7f6903e8dd00 4 rocksdb: Options.avoid_flush_during_shutdown: 0 2017-09-22 09:46:37.942139 7f6903e8dd00 4 rocksdb: Options.delayed_write_rate : 16777216 2017-09-22 09:46:37.942140 7f6903e8dd00 4 rocksdb: Options.max_total_wal_size: 0 2017-09-22 09:46:37.942147 7f6903e8dd00 4 rocksdb: Options.delete_obsolete_files_period_micros: 21600000000 2017-09-22 09:46:37.942149 7f6903e8dd00 4 rocksdb: Options.stats_dump_period_sec: 600 2017-09-22 09:46:37.942150 7f6903e8dd00 4 rocksdb: Compression algorithms supported: 2017-09-22 09:46:37.942152 7f6903e8dd00 4 rocksdb: Snappy supported: 0 2017-09-22 09:46:37.942153 7f6903e8dd00 4 rocksdb: Zlib supported: 0 2017-09-22 09:46:37.942153 7f6903e8dd00 4 rocksdb: Bzip supported: 0 2017-09-22 09:46:37.942154 7f6903e8dd00 4 rocksdb: LZ4 supported: 0 2017-09-22 09:46:37.942154 7f6903e8dd00 4 rocksdb: ZSTD supported: 0 2017-09-22 09:46:37.942155 7f6903e8dd00 4 rocksdb: Fast CRC32 supported: 1 2017-09-22 09:46:37.942377 7f6903e8dd00 4 rocksdb: [/home/zstor/ceph/rpmbuild/BUILD/ceph-12.2.0/src/rocksdb/db/version_set.cc:2609] Recovering from manifest file: MANIFEST-001678 2017-09-22 09:46:37.942435 7f6903e8dd00 4 rocksdb: [/home/zstor/ceph/rpmbuild/BUILD/ceph-12.2.0/src/rocksdb/db/column_family.cc:407] --------------- Options for column family [default]: 2017-09-22 09:46:37.942439 7f6903e8dd00 4 rocksdb: Options.comparator: leveldb.BytewiseComparator 2017-09-22 09:46:37.942445 7f6903e8dd00 4 rocksdb: Options.merge_operator: .T:int64_array.b:bitwise_xor 2017-09-22 09:46:37.942446 7f6903e8dd00 4 rocksdb: Options.compaction_filter: None 2017-09-22 09:46:37.942447 7f6903e8dd00 4 rocksdb: Options.compaction_filter_factory: None 2017-09-22 09:46:37.942447 7f6903e8dd00 4 rocksdb: Options.memtable_factory: SkipListFactory 2017-09-22 09:46:37.942448 7f6903e8dd00 4 rocksdb: Options.table_factory: BlockBasedTable 2017-09-22 09:46:37.942465 7f6903e8dd00 4 rocksdb: table_factory options: flush_block_policy_factory: FlushBlockBySizePolicyFactory (0x7f6905cd2330) cache_index_and_filter_blocks: 1 cache_index_and_filter_blocks_with_high_priority: 1 pin_l0_filter_and_index_blocks_in_cache: 1 index_type: 0 hash_index_allow_collision: 1 checksum: 1 no_block_cache: 0 block_cache: 0x7f6905d17cf8 block_cache_name: LRUCache block_cache_options: capacity : 536870912 num_shard_bits : 4 strict_capacity_limit : 0 high_pri_pool_ratio: 0.000 block_cache_compressed: (nil) persistent_cache: (nil) block_size: 4096 block_size_deviation: 10 block_restart_interval: 16 index_block_restart_interval: 1 filter_policy: rocksdb.BuiltinBloomFilter whole_key_filtering: 1 format_version: 2 2017-09-22 09:46:37.942475 7f6903e8dd00 4 rocksdb: Options.write_buffer_size: 268435456 2017-09-22 09:46:37.942476 7f6903e8dd00 4 rocksdb: Options.max_write_buffer_number: 4 2017-09-22 09:46:37.942477 7f6903e8dd00 4 rocksdb: Options.compression: NoCompression 2017-09-22 09:46:37.942478 7f6903e8dd00 4 rocksdb: Options.bottommost_compression: Disabled 2017-09-22 09:46:37.942479 7f6903e8dd00 4 rocksdb: Options.prefix_extractor: nullptr 2017-09-22 09:46:37.942479 7f6903e8dd00 4 rocksdb: Options.memtable_insert_with_hint_prefix_extractor: nullptr 2017-09-22 09:46:37.942480 7f6903e8dd00 4 rocksdb: Options.num_levels: 7 2017-09-22 09:46:37.942481 7f6903e8dd00 4 rocksdb: Options.min_write_buffer_number_to_merge: 1 2017-09-22 09:46:37.942482 7f6903e8dd00 4 rocksdb: Options.max_write_buffer_number_to_maintain: 0 2017-09-22 09:46:37.942482 7f6903e8dd00 4 rocksdb: Options.compression_opts.window_bits: -14 2017-09-22 09:46:37.942483 7f6903e8dd00 4 rocksdb: Options.compression_opts.level: -1 2017-09-22 09:46:37.942484 7f6903e8dd00 4 rocksdb: Options.compression_opts.strategy: 0 2017-09-22 09:46:37.942484 7f6903e8dd00 4 rocksdb: Options.compression_opts.max_dict_bytes: 0 2017-09-22 09:46:37.942485 7f6903e8dd00 4 rocksdb: Options.level0_file_num_compaction_trigger: 4 2017-09-22 09:46:37.942486 7f6903e8dd00 4 rocksdb: Options.level0_slowdown_writes_trigger: 20 2017-09-22 09:46:37.942492 7f6903e8dd00 4 rocksdb: Options.level0_stop_writes_trigger: 36 2017-09-22 09:46:37.942492 7f6903e8dd00 4 rocksdb: Options.target_file_size_base: 67108864 2017-09-22 09:46:37.942493 7f6903e8dd00 4 rocksdb: Options.target_file_size_multiplier: 1 2017-09-22 09:46:37.942494 7f6903e8dd00 4 rocksdb: Options.max_bytes_for_level_base: 268435456 2017-09-22 09:46:37.942494 7f6903e8dd00 4 rocksdb: Options.level_compaction_dynamic_level_bytes: 0 2017-09-22 09:46:37.942495 7f6903e8dd00 4 rocksdb: Options.max_bytes_for_level_multiplier: 10.000000 2017-09-22 09:46:37.942498 7f6903e8dd00 4 rocksdb: Options.max_bytes_for_level_multiplier_addtl[0]: 1 2017-09-22 09:46:37.942499 7f6903e8dd00 4 rocksdb: Options.max_bytes_for_level_multiplier_addtl[1]: 1 2017-09-22 09:46:37.942499 7f6903e8dd00 4 rocksdb: Options.max_bytes_for_level_multiplier_addtl[2]: 1 2017-09-22 09:46:37.942500 7f6903e8dd00 4 rocksdb: Options.max_bytes_for_level_multiplier_addtl[3]: 1 2017-09-22 09:46:37.942501 7f6903e8dd00 4 rocksdb: Options.max_bytes_for_level_multiplier_addtl[4]: 1 2017-09-22 09:46:37.942501 7f6903e8dd00 4 rocksdb: Options.max_bytes_for_level_multiplier_addtl[5]: 1 2017-09-22 09:46:37.942502 7f6903e8dd00 4 rocksdb: Options.max_bytes_for_level_multiplier_addtl[6]: 1 2017-09-22 09:46:37.942503 7f6903e8dd00 4 rocksdb: Options.max_sequential_skip_in_iterations: 8 2017-09-22 09:46:37.942504 7f6903e8dd00 4 rocksdb: Options.max_compaction_bytes: 1677721600 2017-09-22 09:46:37.942504 7f6903e8dd00 4 rocksdb: Options.arena_block_size: 33554432 2017-09-22 09:46:37.942505 7f6903e8dd00 4 rocksdb: Options.soft_pending_compaction_bytes_limit: 68719476736 2017-09-22 09:46:37.942506 7f6903e8dd00 4 rocksdb: Options.hard_pending_compaction_bytes_limit: 274877906944 2017-09-22 09:46:37.942506 7f6903e8dd00 4 rocksdb: Options.rate_limit_delay_max_milliseconds: 100 2017-09-22 09:46:37.942507 7f6903e8dd00 4 rocksdb: Options.disable_auto_compactions: 0 2017-09-22 09:46:37.942508 7f6903e8dd00 4 rocksdb: Options.compaction_style: kCompactionStyleLevel 2017-09-22 09:46:37.942509 7f6903e8dd00 4 rocksdb: Options.compaction_pri: kByCompensatedSize 2017-09-22 09:46:37.942510 7f6903e8dd00 4 rocksdb: Options.compaction_options_universal.size_ratio: 1 2017-09-22 09:46:37.942511 7f6903e8dd00 4 rocksdb: Options.compaction_options_universal.min_merge_width: 2 2017-09-22 09:46:37.942511 7f6903e8dd00 4 rocksdb: Options.compaction_options_universal.max_merge_width: 4294967295 2017-09-22 09:46:37.942513 7f6903e8dd00 4 rocksdb: Options.compaction_options_universal.max_size_amplification_percent: 200 2017-09-22 09:46:37.942514 7f6903e8dd00 4 rocksdb: Options.compaction_options_universal.compression_size_percent: -1 2017-09-22 09:46:37.942514 7f6903e8dd00 4 rocksdb: Options.compaction_options_fifo.max_table_files_size: 1073741824 2017-09-22 09:46:37.942515 7f6903e8dd00 4 rocksdb: Options.table_properties_collectors: 2017-09-22 09:46:37.942516 7f6903e8dd00 4 rocksdb: Options.inplace_update_support: 0 2017-09-22 09:46:37.942517 7f6903e8dd00 4 rocksdb: Options.inplace_update_num_locks: 10000 2017-09-22 09:46:37.942518 7f6903e8dd00 4 rocksdb: Options.memtable_prefix_bloom_size_ratio: 0.000000 2017-09-22 09:46:37.942519 7f6903e8dd00 4 rocksdb: Options.memtable_huge_page_size: 0 2017-09-22 09:46:37.942519 7f6903e8dd00 4 rocksdb: Options.bloom_locality: 0 2017-09-22 09:46:37.942520 7f6903e8dd00 4 rocksdb: Options.max_successive_merges: 0 2017-09-22 09:46:37.942520 7f6903e8dd00 4 rocksdb: Options.optimize_filters_for_hits: 0 2017-09-22 09:46:37.942521 7f6903e8dd00 4 rocksdb: Options.paranoid_file_checks: 0 2017-09-22 09:46:37.942522 7f6903e8dd00 4 rocksdb: Options.force_consistency_checks: 0 2017-09-22 09:46:37.942523 7f6903e8dd00 4 rocksdb: Options.report_bg_io_stats: 0 2017-09-22 09:46:37.949614 7f6903e8dd00 4 rocksdb: [/home/zstor/ceph/rpmbuild/BUILD/ceph-12.2.0/src/rocksdb/db/version_set.cc:2859] Recovered from manifest file:db/MANIFEST-001678 succeeded,manifest_file_number is 1678, next_file_number is 1680, last_sequence is 19158440, log_number is 0,prev_log_number is 0,max_column_family is 0 2017-09-22 09:46:37.949625 7f6903e8dd00 4 rocksdb: [/home/zstor/ceph/rpmbuild/BUILD/ceph-12.2.0/src/rocksdb/db/version_set.cc:2867] Column family [default] (ID 0), log number is 1677 2017-09-22 09:46:37.949734 7f6903e8dd00 4 rocksdb: EVENT_LOG_v1 {"time_micros": 1506044797949726, "job": 1, "event": "recovery_started", "log_files": [1679]} 2017-09-22 09:46:37.949739 7f6903e8dd00 4 rocksdb: [/home/zstor/ceph/rpmbuild/BUILD/ceph-12.2.0/src/rocksdb/db/db_impl_open.cc:482] Recovering log #1679 mode 0 2017-09-22 09:46:37.949751 7f6903e8dd00 4 rocksdb: [/home/zstor/ceph/rpmbuild/BUILD/ceph-12.2.0/src/rocksdb/db/version_set.cc:2395] Creating manifest 1681 2017-09-22 09:46:37.949843 7f6903e8dd00 10 bitmapalloc:reserve instance 140089048682960 num_used 2940 total 38912 2017-09-22 09:46:37.949848 7f6903e8dd00 10 bitmapalloc:allocate instance 140089048682960 want_size 1048576 alloc_unit 1048576 hint 0 2017-09-22 09:46:37.950197 7f6903e8dd00 10 bitmapalloc:reserve instance 140089048682864 num_used 137 total 6144 2017-09-22 09:46:37.950204 7f6903e8dd00 10 bitmapalloc:allocate instance 140089048682864 want_size 4194304 alloc_unit 1048576 hint 948961280 2017-09-22 09:46:37.950702 7f6903e8dd00 10 bitmapalloc:reserve instance 140089048682960 num_used 2941 total 38912 2017-09-22 09:46:37.950709 7f6903e8dd00 10 bitmapalloc:allocate instance 140089048682960 want_size 1048576 alloc_unit 1048576 hint 0 2017-09-22 09:46:37.951339 7f6903e8dd00 4 rocksdb: EVENT_LOG_v1 {"time_micros": 1506044797951336, "job": 1, "event": "recovery_finished"} 2017-09-22 09:46:37.951444 7f6903e8dd00 4 rocksdb: EVENT_LOG_v1 {"time_micros": 1506044797951441, "job": 2, "event": "table_file_deletion", "file_number": 1682} 2017-09-22 09:46:37.951716 7f6903e8dd00 10 bitmapalloc:release 0x2b00000~200000 2017-09-22 09:46:37.951721 7f6903e8dd00 20 bitmapalloc:insert_free instance 140089048682960 off 0x2b00000 len 0x200000 2017-09-22 09:46:37.951723 7f6903e8dd00 10 bitmapalloc:release 0x1bf29400000~4100000 2017-09-22 09:46:37.951724 7f6903e8dd00 20 bitmapalloc:insert_free instance 140089048683008 off 0x1bf29400000 len 0x4100000 2017-09-22 09:46:37.951831 7f6903e8dd00 10 bitmapalloc:reserve instance 140089048682960 num_used 2940 total 38912 2017-09-22 09:46:37.951835 7f6903e8dd00 10 bitmapalloc:allocate instance 140089048682960 want_size 1048576 alloc_unit 1048576 hint 0 2017-09-22 09:46:37.954221 7f6903e8dd00 4 rocksdb: [/home/zstor/ceph/rpmbuild/BUILD/ceph-12.2.0/src/rocksdb/db/db_impl_open.cc:1063] DB pointer 0x7f69061bc000 2017-09-22 09:46:37.954257 7f6903e8dd00 1 bluestore(/var/lib/ceph/osd/ceph-19/) _open_db opened rocksdb path db options compression=kNoCompression,max_write_buffer_number=4,min_write_buffer_number_to_merge=1,recycle_log_file_num=4,write_buffer_size=268435456,writable_file_max_buffer_size=0,compaction_readahead_size=2097152 2017-09-22 09:46:37.954391 7f68eb7c8700 3 rocksdb: [/home/zstor/ceph/rpmbuild/BUILD/ceph-12.2.0/src/rocksdb/db/db_impl.cc:447] ------- DUMPING STATS ------- 2017-09-22 09:46:37.954396 7f68eb7c8700 3 rocksdb: [/home/zstor/ceph/rpmbuild/BUILD/ceph-12.2.0/src/rocksdb/db/db_impl.cc:448] ** DB Stats ** Uptime(secs): 0.0 total, 0.0 interval Cumulative writes: 0 writes, 0 keys, 0 commit groups, 0.0 writes per commit group, ingest: 0.00 GB, 0.00 MB/s Cumulative WAL: 0 writes, 0 syncs, 0.00 writes per sync, written: 0.00 GB, 0.00 MB/s Cumulative stall: 00:00:0.000 H:M:S, 0.0 percent Interval writes: 0 writes, 0 keys, 0 commit groups, 0.0 writes per commit group, ingest: 0.00 MB, 0.00 MB/s Interval WAL: 0 writes, 0 syncs, 0.00 writes per sync, written: 0.00 MB, 0.00 MB/s Interval stall: 00:00:0.000 H:M:S, 0.0 percent ** Compaction Stats [default] ** Level Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop ---------------------------------------------------------------------------------------------------------------------------------------------------------- L0 1/0 36.81 MB 0.2 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0 0 0.000 0 0 L1 4/0 252.59 MB 1.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0 0 0.000 0 0 L2 44/0 2.54 GB 1.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0 0 0.000 0 0 L3 13/0 842.16 MB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0 0 0.000 0 0 Sum 62/0 3.65 GB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0 0 0.000 0 0 Int 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0 0 0.000 0 0 Uptime(secs): 0.0 total, 0.0 interval Flush(GB): cumulative 0.000, interval 0.000 AddFile(GB): cumulative 0.000, interval 0.000 AddFile(Total Files): cumulative 0, interval 0 AddFile(L0 Files): cumulative 0, interval 0 AddFile(Keys): cumulative 0, interval 0 Cumulative compaction: 0.00 GB write, 0.00 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds Interval compaction: 0.00 GB write, 0.00 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds Stalls(count): 0 level0_slowdown, 0 level0_slowdown_with_compaction, 0 level0_numfiles, 0 level0_numfiles_with_compaction, 0 stop for pending_compaction_bytes, 0 slowdown for pending_compaction_bytes, 0 memtable_compaction, 0 memtable_slowdown, interval 0 total count ** File Read Latency Histogram By Level [default] ** ** Compaction Stats [default] ** Level Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop ---------------------------------------------------------------------------------------------------------------------------------------------------------- L0 1/0 36.81 MB 0.2 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0 0 0.000 0 0 L1 4/0 252.59 MB 1.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0 0 0.000 0 0 L2 44/0 2.54 GB 1.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0 0 0.000 0 0 L3 13/0 842.16 MB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0 0 0.000 0 0 Sum 62/0 3.65 GB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0 0 0.000 0 0 Int 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0 0 0.000 0 0 Uptime(secs): 0.0 total, 0.0 interval Flush(GB): cumulative 0.000, interval 0.000 AddFile(GB): cumulative 0.000, interval 0.000 AddFile(Total Files): cumulative 0, interval 0 AddFile(L0 Files): cumulative 0, interval 0 AddFile(Keys): cumulative 0, interval 0 Cumulative compaction: 0.00 GB write, 0.00 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds Interval compaction: 0.00 GB write, 0.00 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds Stalls(count): 0 level0_slowdown, 0 level0_slowdown_with_compaction, 0 level0_numfiles, 0 level0_numfiles_with_compaction, 0 stop for pending_compaction_bytes, 0 slowdown for pending_compaction_bytes, 0 memtable_compaction, 0 memtable_slowdown, interval 0 total count ** File Read Latency Histogram By Level [default] ** 2017-09-22 09:46:37.954426 7f68eb7c8700 4 rocksdb: [/home/zstor/ceph/rpmbuild/BUILD/ceph-12.2.0/src/rocksdb/db/compaction_picker.cc:461] [default] Expanding@2 1+2(69564368+145598984 bytes) to 2+2 (81603618+145598984bytes) 2017-09-22 09:46:37.954462 7f68eb7c8700 4 rocksdb: [/home/zstor/ceph/rpmbuild/BUILD/ceph-12.2.0/src/rocksdb/db/compaction_job.cc:1403] [default] [JOB 3] Compacting 2@2 + 2@3 files to L3, score 1.02 2017-09-22 09:46:37.954476 7f68eb7c8700 4 rocksdb: [/home/zstor/ceph/rpmbuild/BUILD/ceph-12.2.0/src/rocksdb/db/compaction_job.cc:1407] [default] Compaction start summary: Base version 2 Base level 2, inputs: [1528(66MB) 1529(11MB)], [628(69MB) 680(69MB)] 2017-09-22 09:46:37.954493 7f68eb7c8700 4 rocksdb: EVENT_LOG_v1 {"time_micros": 1506044797954478, "job": 3, "event": "compaction_started", "files_L2": [1528, 1529], "files_L3": [628, 680], "score": 1.01793, "input_data_size": 227202602} 2017-09-22 09:46:37.954581 7f6903e8dd00 10 bluestore(/var/lib/ceph/osd/ceph-19/) _open_super_meta old nid_max 712666 2017-09-22 09:46:37.954597 7f6903e8dd00 10 bluestore(/var/lib/ceph/osd/ceph-19/) _open_super_meta old blobid_max 163840 2017-09-22 09:46:38.063809 7f6903e8dd00 10 bluestore(/var/lib/ceph/osd/ceph-19/) _open_super_meta freelist_type bitmap 2017-09-22 09:46:38.063862 7f6903e8dd00 10 bluestore(/var/lib/ceph/osd/ceph-19/) _open_super_meta bluefs_extents 0x[1bf1c600000~2542600000] 2017-09-22 09:46:38.063884 7f6903e8dd00 10 bluestore(/var/lib/ceph/osd/ceph-19/) _open_super_meta ondisk_format 2 compat_ondisk_format 2 2017-09-22 09:46:38.063899 7f6903e8dd00 10 bluestore(/var/lib/ceph/osd/ceph-19/) _open_super_meta min_alloc_size 0x400000 2017-09-22 09:46:38.064313 7f6903e8dd00 10 bluestore(/var/lib/ceph/osd/ceph-19/) _set_alloc_sizes min_alloc_size 0x400000 order  max_alloc_size 0x0 prefer_deferred_size 0x0 deferred_batch_ops 16 2017-09-22 09:46:38.064322 7f6903e8dd00 10 bluestore(/var/lib/ceph/osd/ceph-19/) _set_throttle_params throttle_cost_per_io 4000 2017-09-22 09:46:38.064325 7f6903e8dd00 10 bluestore(/var/lib/ceph/osd/ceph-19/) _set_csum csum_type crc32c 2017-09-22 09:46:38.064338 7f6903e8dd00 10 bluestore(/var/lib/ceph/osd/ceph-19/) _set_compression compression mode set to 'none', ignore other compression setttings 2017-09-22 09:46:38.064340 7f6903e8dd00 10 bluestore(/var/lib/ceph/osd/ceph-19/) _set_blob_size max_blob_size 0x400000 2017-09-22 09:46:38.064349 7f6903e8dd00 1 freelist init 2017-09-22 09:46:38.103853 7f6903e8dd00 10 freelist init blocks 0x3a37b300 2017-09-22 09:46:38.103882 7f6903e8dd00 10 freelist init blocks_per_key 0x80 2017-09-22 09:46:38.103886 7f6903e8dd00 10 freelist init bytes_per_block 0x1000 2017-09-22 09:46:38.103890 7f6903e8dd00 10 freelist init size 0x3a37b2cf000 2017-09-22 09:46:38.103892 7f6903e8dd00 10 freelist init size 0x3a37b2cf000 bytes_per_block 0x1000 blocks 0x3a37b300 blocks_per_key 0x80 2017-09-22 09:46:38.103906 7f6903e8dd00 10 freelist _init_misc bytes_per_key 0x80000, key_mask 0xfffffffffff80000 2017-09-22 09:46:38.104606 7f6903e8dd00 10 bitmapalloc:BitMapAllocator instance 140089048693184 size 0x3a37b2cf000 2017-09-22 09:46:38.104613 7f6903e8dd00 1 bluestore(/var/lib/ceph/osd/ceph-19/) _open_alloc opening allocation metadata 2017-09-22 09:46:38.104619 7f6903e8dd00 10 freelist enumerate_next start 2017-09-22 09:46:38.104683 7f6903e8dd00 10 freelist enumerate_next 0x2000~3fe000 2017-09-22 09:46:38.104688 7f6903e8dd00 10 bitmapalloc:init_add_free instance 140089048693184 offset 0x2000 length 0x3fe000 2017-09-22 09:46:38.104691 7f6903e8dd00 20 bitmapalloc:insert_free instance 140089048693184 off 0x400000 len 0x0 2017-09-22 09:46:38.683243 7f68eb7c8700 10 bitmapalloc:reserve instance 140089048683008 num_used 3663583 total 3815346 2017-09-22 09:46:38.683254 7f68eb7c8700 10 bitmapalloc:allocate instance 140089048683008 want_size 68157440 alloc_unit 1048576 hint 0 2017-09-22 09:46:39.117189 7f68eb7c8700 10 bitmapalloc:release 0xd00000~100000 2017-09-22 09:46:39.117201 7f68eb7c8700 20 bitmapalloc:insert_free instance 140089048682960 off 0xd00000 len 0x100000 2017-09-22 09:46:39.117271 7f68eb7c8700 3 rocksdb: [/home/zstor/ceph/rpmbuild/BUILD/ceph-12.2.0/src/rocksdb/db/db_impl_compaction_flush.cc:1591] Compaction error: Corruption: block checksum mismatch 2017-09-22 09:46:39.117284 7f68eb7c8700 4 rocksdb: (Original Log Time 2017/09/22-09:46:39.117233) [/home/zstor/ceph/rpmbuild/BUILD/ceph-12.2.0/src/rocksdb/db/compaction_job.cc:621] [default] compacted to: base level 1 max bytes base 268435456 files[1 4 44 13 0 0 0] max score 0.99, MB/sec: 195.4 rd, 57.7 wr, level 3, files in(2, 2) out(1) MB in(77.8, 138.9) out(64.0), read-write-amplify(3.6) write-amplify(0.8) Corruption: block checksum mismatch, records in: 5211814, records dropped: 528 2017-09-22 09:46:39.117290 7f68eb7c8700 4 rocksdb: (Original Log Time 2017/09/22-09:46:39.117264) EVENT_LOG_v1 {"time_micros": 1506044799117253, "job": 3, "event": "compaction_finished", "compaction_time_micros": 1162712, "output_level": 3, "num_output_files": 1, "total_output_size": 67110452, "num_input_records": 558271, "num_output_records": 557743, "num_subcompactions": 1, "num_single_delete_mismatches": 0, "num_single_delete_fallthrough": 0, "lsm_state": [1, 4, 44, 13, 0, 0, 0]} 2017-09-22 09:46:39.117293 7f68eb7c8700 2 rocksdb: [/home/zstor/ceph/rpmbuild/BUILD/ceph-12.2.0/src/rocksdb/db/db_impl_compaction_flush.cc:1275] Waiting after background compaction error: Corruption: block checksum mismatch, Accumulated background error counts: 1 2017-09-22 09:46:40.117512 7f68eb7c8700 4 rocksdb: EVENT_LOG_v1 {"time_micros": 1506044800117501, "job": 3, "event": "table_file_deletion", "file_number": 1685} 2017-09-22 09:46:45.162235 7f6903e8dd00 10 freelist enumerate_next 0x1bf1c800000~2542400000 2017-09-22 09:46:45.162261 7f6903e8dd00 10 bitmapalloc:init_add_free instance 140089048693184 offset 0x1bf1c800000 length 0x2542400000 2017-09-22 09:46:45.162263 7f6903e8dd00 20 bitmapalloc:insert_free instance 140089048693184 off 0x1bf1c800000 len 0x2542400000 2017-09-22 09:46:50.184371 7f6903e8dd00 10 freelist enumerate_next 0x2cd60c00000~9f400000 2017-09-22 09:46:50.184386 7f6903e8dd00 10 bitmapalloc:init_add_free instance 140089048693184 offset 0x2cd60c00000 length 0x9f400000 2017-09-22 09:46:50.184388 7f6903e8dd00 20 bitmapalloc:insert_free instance 140089048693184 off 0x2cd60c00000 len 0x9f400000 2017-09-22 09:46:50.184443 7f6903e8dd00 10 freelist enumerate_next 0x2ce00800000~d57aacf000 2017-09-22 09:46:50.184444 7f6903e8dd00 10 bitmapalloc:init_add_free instance 140089048693184 offset 0x2ce00800000 length 0xd57aacf000 2017-09-22 09:46:50.184453 7f6903e8dd00 20 bitmapalloc:insert_free instance 140089048693184 off 0x2ce00800000 len 0xd57a800000 2017-09-22 09:46:50.184568 7f6903e8dd00 10 freelist enumerate_next end 2017-09-22 09:46:50.184578 7f6903e8dd00 1 bluestore(/var/lib/ceph/osd/ceph-19/) _open_alloc loaded 1005 G in 4 extents 2017-09-22 09:46:50.184591 7f6903e8dd00 10 bitmapalloc:init_rm_free instance 140089048693184 offset 0x1bf1c600000 length 0x2542600000 2017-09-22 09:46:50.184614 7f6903e8dd00 10 bluestore(/var/lib/ceph/osd/ceph-19/) _open_alloc marked bluefs_extents 0x[1bf1c600000~2542600000] as allocated 2017-09-22 09:46:50.184965 7f6903e8dd00 20 bluestore(/var/lib/ceph/osd/ceph-19/) _open_collections opened 3.0_head 0x7f6905d1a8c0 2017-09-22 09:46:50.184999 7f6903e8dd00 20 bluestore(/var/lib/ceph/osd/ceph-19/) _open_collections opened 3.100_head 0x7f690611aa00 2017-09-22 09:46:50.185025 7f6903e8dd00 20 bluestore(/var/lib/ceph/osd/ceph-19/) _open_collections opened 3.101_head 0x7f690611b800 2017-09-22 09:46:50.185031 7f6903e8dd00 20 bluestore(/var/lib/ceph/osd/ceph-19/) _open_collections opened 3.103_head 0x7f690611b480 2017-09-22 09:46:50.185036 7f6903e8dd00 20 bluestore(/var/lib/ceph/osd/ceph-19/) _open_collections opened 3.104_head 0x7f690611a140 2017-09-22 09:46:50.185039 7f6903e8dd00 20 bluestore(/var/lib/ceph/osd/ceph-19/) _open_collections opened 3.105_head 0x7f6906119c00 2017-09-22 09:46:50.185048 7f6903e8dd00 20 bluestore(/var/lib/ceph/osd/ceph-19/) _open_collections opened 3.108_head 0x7f69061a5a40 2017-09-22 09:46:50.185053 7f6903e8dd00 20 bluestore(/var/lib/ceph/osd/ceph-19/) _open_collections opened 3.10_head 0x7f690611bd40 2017-09-22 09:46:50.185057 7f6903e8dd00 20 bluestore(/var/lib/ceph/osd/ceph-19/) _open_collections opened 3.10b_head 0x7f690611b100 2017-09-22 09:46:50.185061 7f6903e8dd00 20 bluestore(/var/lib/ceph/osd/ceph-19/) _open_collections opened 3.10f_head 0x7f690611af40 2017-09-22 09:46:50.185064 7f6903e8dd00 20 bluestore(/var/lib/ceph/osd/ceph-19/) _open_collections opened 3.110_head 0x7f690611a680 2017-09-22 09:46:50.185075 7f6903e8dd00 20 bluestore(/var/lib/ceph/osd/ceph-19/) _open_collections opened 3.114_head 0x7f690611a4c0 2017-09-22 09:46:50.185087 7f6903e8dd00 20 bluestore(/var/lib/ceph/osd/ceph-19/) _open_collections opened 3.115_head 0x7f69061a5880 2017-09-22 09:46:50.185091 7f6903e8dd00 20 bluestore(/var/lib/ceph/osd/ceph-19/) _open_collections opened 3.116_head 0x7f69061a5340 2017-09-22 09:46:50.185094 7f6903e8dd00 20 bluestore(/var/lib/ceph/osd/ceph-19/) _open_collections opened 3.118_head 0x7f69061a5180 2017-09-22 09:46:50.185096 7f6903e8dd00 20 bluestore(/var/lib/ceph/osd/ceph-19/) _open_collections opened 3.11a_head 0x7f69061a4e00 2017-09-22 09:46:50.185099 7f6903e8dd00 20 bluestore(/var/lib/ceph/osd/ceph-19/) _open_collections opened 3.11c_head 0x7f69061a4c40 2017-09-22 09:46:50.185102 7f6903e8dd00 20 bluestore(/var/lib/ceph/osd/ceph-19/) _open_collections opened 3.11f_head 0x7f69061a48c0 2017-09-22 09:46:50.185105 7f6903e8dd00 20 bluestore(/var/lib/ceph/osd/ceph-19/) _open_collections opened 3.12_head 0x7f69061a4700 2017-09-22 09:46:50.185113 7f6903e8dd00 20 bluestore(/var/lib/ceph/osd/ceph-19/) _open_collections opened 3.16_head 0x7f69061a4380 2017-09-22 09:46:50.185116 7f6903e8dd00 20 bluestore(/var/lib/ceph/osd/ceph-19/) _open_collections opened 3.18_head 0x7f69061a41c0 2017-09-22 09:46:50.185119 7f6903e8dd00 20 bluestore(/var/lib/ceph/osd/ceph-19/) _open_collections opened 3.1_head 0x7f69061a5dc0 2017-09-22 09:46:50.185122 7f6903e8dd00 20 bluestore(/var/lib/ceph/osd/ceph-19/) _open_collections opened 3.1b_head 0x7f69061a5f80 2017-09-22 09:46:50.185127 7f6903e8dd00 20 bluestore(/var/lib/ceph/osd/ceph-19/) _open_collections opened 3.1f_head 0x7f69061a6140 2017-09-22 09:46:50.185145 7f6903e8dd00 20 bluestore(/var/lib/ceph/osd/ceph-19/) _open_collections opened 3.22_head 0x7f69061a6300 2017-09-22 09:46:50.185149 7f6903e8dd00 20 bluestore(/var/lib/ceph/osd/ceph-19/) _open_collections opened 3.25_head 0x7f69061a64c0 2017-09-22 09:46:50.185152 7f6903e8dd00 20 bluestore(/var/lib/ceph/osd/ceph-19/) _open_collections opened 3.2_head 0x7f69061a6680 2017-09-22 09:46:50.185160 7f6903e8dd00 20 bluestore(/var/lib/ceph/osd/ceph-19/) _open_collections opened 3.2a_head 0x7f69061a6840 2017-09-22 09:46:50.185163 7f6903e8dd00 20 bluestore(/var/lib/ceph/osd/ceph-19/) _open_collections opened 3.2d_head 0x7f69061a6a00 2017-09-22 09:46:50.185166 7f6903e8dd00 20 bluestore(/var/lib/ceph/osd/ceph-19/) _open_collections opened 3.2e_head 0x7f69061a6bc0 2017-09-22 09:46:50.185169 7f6903e8dd00 20 bluestore(/var/lib/ceph/osd/ceph-19/) _open_collections opened 3.2f_head 0x7f69061a6d80 2017-09-22 09:46:50.185172 7f6903e8dd00 20 bluestore(/var/lib/ceph/osd/ceph-19/) _open_collections opened 3.30_head 0x7f69061a6f40 2017-09-22 09:46:50.185174 7f6903e8dd00 20 bluestore(/var/lib/ceph/osd/ceph-19/) _open_collections opened 3.31_head 0x7f69061a7100 2017-09-22 09:46:50.185177 7f6903e8dd00 20 bluestore(/var/lib/ceph/osd/ceph-19/) _open_collections opened 3.34_head 0x7f69061a72c0 2017-09-22 09:46:50.185187 7f6903e8dd00 20 bluestore(/var/lib/ceph/osd/ceph-19/) _open_collections opened 3.37_head 0x7f69061a7480 2017-09-22 09:46:50.185196 7f6903e8dd00 20 bluestore(/var/lib/ceph/osd/ceph-19/) _open_collections opened 3.38_head 0x7f69061a7640 2017-09-22 09:46:50.185199 7f6903e8dd00 20 bluestore(/var/lib/ceph/osd/ceph-19/) _open_collections opened 3.3a_head 0x7f69061a7800 2017-09-22 09:46:50.185202 7f6903e8dd00 20 bluestore(/var/lib/ceph/osd/ceph-19/) _open_collections opened 3.3b_head 0x7f69061a79c0 2017-09-22 09:46:50.185205 7f6903e8dd00 20 bluestore(/var/lib/ceph/osd/ceph-19/) _open_collections opened 3.3d_head 0x7f69061a7b80 2017-09-22 09:46:50.185208 7f6903e8dd00 20 bluestore(/var/lib/ceph/osd/ceph-19/) _open_collections opened 3.41_head 0x7f69061a7d40 2017-09-22 09:46:50.185218 7f6903e8dd00 20 bluestore(/var/lib/ceph/osd/ceph-19/) _open_collections opened 3.45_head 0x7f6911f82000 2017-09-22 09:46:50.185221 7f6903e8dd00 20 bluestore(/var/lib/ceph/osd/ceph-19/) _open_collections opened 3.46_head 0x7f6911f821c0 2017-09-22 09:46:50.185224 7f6903e8dd00 20 bluestore(/var/lib/ceph/osd/ceph-19/) _open_collections opened 3.4c_head 0x7f6911f82380 2017-09-22 09:46:50.185227 7f6903e8dd00 20 bluestore(/var/lib/ceph/osd/ceph-19/) _open_collections opened 3.4f_head 0x7f6911f82540 2017-09-22 09:46:50.185344 7f6903e8dd00 20 bluestore(/var/lib/ceph/osd/ceph-19/) _open_collections opened 3.51_head 0x7f6911f82700 2017-09-22 09:46:50.185348 7f6903e8dd00 20 bluestore(/var/lib/ceph/osd/ceph-19/) _open_collections opened 3.53_head 0x7f6911f828c0 2017-09-22 09:46:50.185351 7f6903e8dd00 20 bluestore(/var/lib/ceph/osd/ceph-19/) _open_collections opened 3.56_head 0x7f6911f82a80 2017-09-22 09:46:50.185355 7f6903e8dd00 20 bluestore(/var/lib/ceph/osd/ceph-19/) _open_collections opened 3.57_head 0x7f6911f82c40 2017-09-22 09:46:50.185357 7f6903e8dd00 20 bluestore(/var/lib/ceph/osd/ceph-19/) _open_collections opened 3.59_head 0x7f6911f82e00 2017-09-22 09:46:50.185360 7f6903e8dd00 20 bluestore(/var/lib/ceph/osd/ceph-19/) _open_collections opened 3.5b_head 0x7f6911f82fc0 2017-09-22 09:46:50.185363 7f6903e8dd00 20 bluestore(/var/lib/ceph/osd/ceph-19/) _open_collections opened 3.5f_head 0x7f6911f83180 2017-09-22 09:46:50.185366 7f6903e8dd00 20 bluestore(/var/lib/ceph/osd/ceph-19/) _open_collections opened 3.60_head 0x7f6911f83340 2017-09-22 09:46:50.185369 7f6903e8dd00 20 bluestore(/var/lib/ceph/osd/ceph-19/) _open_collections opened 3.62_head 0x7f6911f83500 2017-09-22 09:46:50.185372 7f6903e8dd00 20 bluestore(/var/lib/ceph/osd/ceph-19/) _open_collections opened 3.68_head 0x7f6911f836c0 2017-09-22 09:46:50.185374 7f6903e8dd00 20 bluestore(/var/lib/ceph/osd/ceph-19/) _open_collections opened 3.69_head 0x7f6911f83880 2017-09-22 09:46:50.185377 7f6903e8dd00 20 bluestore(/var/lib/ceph/osd/ceph-19/) _open_collections opened 3.6a_head 0x7f6911f83a40 2017-09-22 09:46:50.185380 7f6903e8dd00 20 bluestore(/var/lib/ceph/osd/ceph-19/) _open_collections opened 3.6d_head 0x7f6911f83c00 2017-09-22 09:46:50.185383 7f6903e8dd00 20 bluestore(/var/lib/ceph/osd/ceph-19/) _open_collections opened 3.6e_head 0x7f6911f83dc0 2017-09-22 09:46:50.185385 7f6903e8dd00 20 bluestore(/var/lib/ceph/osd/ceph-19/) _open_collections opened 3.71_head 0x7f6911f83f80 2017-09-22 09:46:50.185388 7f6903e8dd00 20 bluestore(/var/lib/ceph/osd/ceph-19/) _open_collections opened 3.74_head 0x7f6911f84140 2017-09-22 09:46:50.185391 7f6903e8dd00 20 bluestore(/var/lib/ceph/osd/ceph-19/) _open_collections opened 3.75_head 0x7f6911f84300 2017-09-22 09:46:50.185394 7f6903e8dd00 20 bluestore(/var/lib/ceph/osd/ceph-19/) _open_collections opened 3.77_head 0x7f6911f844c0 2017-09-22 09:46:50.185396 7f6903e8dd00 20 bluestore(/var/lib/ceph/osd/ceph-19/) _open_collections opened 3.80_head 0x7f6911f84680 2017-09-22 09:46:50.185399 7f6903e8dd00 20 bluestore(/var/lib/ceph/osd/ceph-19/) _open_collections opened 3.84_head 0x7f6911f84840 2017-09-22 09:46:50.185402 7f6903e8dd00 20 bluestore(/var/lib/ceph/osd/ceph-19/) _open_collections opened 3.86_head 0x7f6911f84a00 2017-09-22 09:46:50.185405 7f6903e8dd00 20 bluestore(/var/lib/ceph/osd/ceph-19/) _open_collections opened 3.87_head 0x7f6911f84bc0 2017-09-22 09:46:50.185408 7f6903e8dd00 20 bluestore(/var/lib/ceph/osd/ceph-19/) _open_collections opened 3.89_head 0x7f6911f84d80 2017-09-22 09:46:50.185410 7f6903e8dd00 20 bluestore(/var/lib/ceph/osd/ceph-19/) _open_collections opened 3.8f_head 0x7f6911f84f40 2017-09-22 09:46:50.185413 7f6903e8dd00 20 bluestore(/var/lib/ceph/osd/ceph-19/) _open_collections opened 3.90_head 0x7f6911f85100 2017-09-22 09:46:50.185415 7f6903e8dd00 20 bluestore(/var/lib/ceph/osd/ceph-19/) _open_collections opened 3.91_head 0x7f6911f852c0 2017-09-22 09:46:50.185418 7f6903e8dd00 20 bluestore(/var/lib/ceph/osd/ceph-19/) _open_collections opened 3.95_head 0x7f6911f85480 2017-09-22 09:46:50.185428 7f6903e8dd00 20 bluestore(/var/lib/ceph/osd/ceph-19/) _open_collections opened 3.96_head 0x7f6911f85640 2017-09-22 09:46:50.185431 7f6903e8dd00 20 bluestore(/var/lib/ceph/osd/ceph-19/) _open_collections opened 3.99_head 0x7f6911f85800 2017-09-22 09:46:50.185434 7f6903e8dd00 20 bluestore(/var/lib/ceph/osd/ceph-19/) _open_collections opened 3.9_head 0x7f6911f859c0 2017-09-22 09:46:50.185437 7f6903e8dd00 20 bluestore(/var/lib/ceph/osd/ceph-19/) _open_collections opened 3.a1_head 0x7f6911f85b80 2017-09-22 09:46:50.185439 7f6903e8dd00 20 bluestore(/var/lib/ceph/osd/ceph-19/) _open_collections opened 3.a3_head 0x7f6911f85d40 2017-09-22 09:46:50.185452 7f6903e8dd00 20 bluestore(/var/lib/ceph/osd/ceph-19/) _open_collections opened 3.a6_head 0x7f6908d06000 2017-09-22 09:46:50.185455 7f6903e8dd00 20 bluestore(/var/lib/ceph/osd/ceph-19/) _open_collections opened 3.a7_head 0x7f6908d061c0 2017-09-22 09:46:50.185458 7f6903e8dd00 20 bluestore(/var/lib/ceph/osd/ceph-19/) _open_collections opened 3.a_head 0x7f6908d06380 2017-09-22 09:46:50.185460 7f6903e8dd00 20 bluestore(/var/lib/ceph/osd/ceph-19/) _open_collections opened 3.ac_head 0x7f6908d06540 2017-09-22 09:46:50.185463 7f6903e8dd00 20 bluestore(/var/lib/ceph/osd/ceph-19/) _open_collections opened 3.ad_head 0x7f6908d06700 2017-09-22 09:46:50.185466 7f6903e8dd00 20 bluestore(/var/lib/ceph/osd/ceph-19/) _open_collections opened 3.b2_head 0x7f6908d068c0 2017-09-22 09:46:50.185468 7f6903e8dd00 20 bluestore(/var/lib/ceph/osd/ceph-19/) _open_collections opened 3.b3_head 0x7f6908d06a80 2017-09-22 09:46:50.185471 7f6903e8dd00 20 bluestore(/var/lib/ceph/osd/ceph-19/) _open_collections opened 3.b8_head 0x7f6908d06c40 2017-09-22 09:46:50.185474 7f6903e8dd00 20 bluestore(/var/lib/ceph/osd/ceph-19/) _open_collections opened 3.b9_head 0x7f6908d06e00 2017-09-22 09:46:50.185477 7f6903e8dd00 20 bluestore(/var/lib/ceph/osd/ceph-19/) _open_collections opened 3.be_head 0x7f6908d06fc0 2017-09-22 09:46:50.185479 7f6903e8dd00 20 bluestore(/var/lib/ceph/osd/ceph-19/) _open_collections opened 3.bf_head 0x7f6908d07180 2017-09-22 09:46:50.185482 7f6903e8dd00 20 bluestore(/var/lib/ceph/osd/ceph-19/) _open_collections opened 3.c3_head 0x7f6908d07340 2017-09-22 09:46:50.185485 7f6903e8dd00 20 bluestore(/var/lib/ceph/osd/ceph-19/) _open_collections opened 3.c6_head 0x7f6908d07500 2017-09-22 09:46:50.185492 7f6903e8dd00 20 bluestore(/var/lib/ceph/osd/ceph-19/) _open_collections opened 3.cb_head 0x7f6908d076c0 2017-09-22 09:46:50.185495 7f6903e8dd00 20 bluestore(/var/lib/ceph/osd/ceph-19/) _open_collections opened 3.d0_head 0x7f6908d07880 2017-09-22 09:46:50.185498 7f6903e8dd00 20 bluestore(/var/lib/ceph/osd/ceph-19/) _open_collections opened 3.d1_head 0x7f6908d07a40 2017-09-22 09:46:50.185507 7f6903e8dd00 20 bluestore(/var/lib/ceph/osd/ceph-19/) _open_collections opened 3.e0_head 0x7f6908d07c00 2017-09-22 09:46:50.185510 7f6903e8dd00 20 bluestore(/var/lib/ceph/osd/ceph-19/) _open_collections opened 3.e3_head 0x7f6908d07dc0 2017-09-22 09:46:50.185513 7f6903e8dd00 20 bluestore(/var/lib/ceph/osd/ceph-19/) _open_collections opened 3.e6_head 0x7f6908d07f80 2017-09-22 09:46:50.185515 7f6903e8dd00 20 bluestore(/var/lib/ceph/osd/ceph-19/) _open_collections opened 3.e8_head 0x7f6908d08140 2017-09-22 09:46:50.185518 7f6903e8dd00 20 bluestore(/var/lib/ceph/osd/ceph-19/) _open_collections opened 3.ec_head 0x7f6908d08300 2017-09-22 09:46:50.185529 7f6903e8dd00 20 bluestore(/var/lib/ceph/osd/ceph-19/) _open_collections opened 3.ef_head 0x7f6908d084c0 2017-09-22 09:46:50.185532 7f6903e8dd00 20 bluestore(/var/lib/ceph/osd/ceph-19/) _open_collections opened 3.f3_head 0x7f6908d08680 2017-09-22 09:46:50.185534 7f6903e8dd00 20 bluestore(/var/lib/ceph/osd/ceph-19/) _open_collections opened 3.f4_head 0x7f6908d08840 2017-09-22 09:46:50.185537 7f6903e8dd00 20 bluestore(/var/lib/ceph/osd/ceph-19/) _open_collections opened 3.f6_head 0x7f6908d08a00 2017-09-22 09:46:50.185539 7f6903e8dd00 20 bluestore(/var/lib/ceph/osd/ceph-19/) _open_collections opened 3.f9_head 0x7f6908d08bc0 2017-09-22 09:46:50.185542 7f6903e8dd00 20 bluestore(/var/lib/ceph/osd/ceph-19/) _open_collections opened 3.fa_head 0x7f6908d08d80 2017-09-22 09:46:50.185544 7f6903e8dd00 20 bluestore(/var/lib/ceph/osd/ceph-19/) _open_collections opened 3.fd_head 0x7f6908d08f40 2017-09-22 09:46:50.185547 7f6903e8dd00 20 bluestore(/var/lib/ceph/osd/ceph-19/) _open_collections opened 3.fe_head 0x7f6908d09100 2017-09-22 09:46:50.185550 7f6903e8dd00 20 bluestore(/var/lib/ceph/osd/ceph-19/) _open_collections opened meta 0x7f6908d092c0 2017-09-22 09:46:50.185624 7f6903e8dd00 10 bluestore(/var/lib/ceph/osd/ceph-19/) _kv_start 2017-09-22 09:46:50.185686 7f68ebfc9700 10 trim shard target 2560 M meta/data ratios 0.833333 + 1.49012e-08 (2133 M + 40 ), current 0 (0 + 0 ) 2017-09-22 09:46:50.185939 7f6903e8dd00 10 bluestore(/var/lib/ceph/osd/ceph-19/) _deferred_replay start 2017-09-22 09:46:50.185949 7f68f07d2700 10 bluestore(/var/lib/ceph/osd/ceph-19/) _kv_sync_thread start 2017-09-22 09:46:50.185965 7f68f07d2700 20 bluestore(/var/lib/ceph/osd/ceph-19/) _kv_sync_thread sleep 2017-09-22 09:46:50.185971 7f68effd1700 10 bluestore(/var/lib/ceph/osd/ceph-19/) _kv_finalize_thread start 2017-09-22 09:46:50.185976 7f68effd1700 20 bluestore(/var/lib/ceph/osd/ceph-19/) _kv_finalize_thread sleep 2017-09-22 09:46:50.185972 7f6903e8dd00 20 bluestore(/var/lib/ceph/osd/ceph-19/) _deferred_replay replay 0x000000000000052a 2017-09-22 09:46:50.186033 7f6903e8dd00 20 bluestore(/var/lib/ceph/osd/ceph-19/) _txc_create osr 0x7f6906126060 = 0x7f6905fb0dc0 seq 1 2017-09-22 09:46:50.186041 7f6903e8dd00 10 bluestore(/var/lib/ceph/osd/ceph-19/) _txc_state_proc txc 0x7f6905fb0dc0 kv_done 2017-09-22 09:46:50.186044 7f6903e8dd00 20 bluestore(/var/lib/ceph/osd/ceph-19/) _deferred_queue txc 0x7f6905fb0dc0 osr 0x7f6906126060 2017-09-22 09:46:50.186080 7f6903e8dd00 20 bluestore.DeferredBatch(0x7f690616b560) prepare_write seq 1322 0x2cc8fc04000~2000 crc c2a954b4 2017-09-22 09:46:50.186112 7f6903e8dd00 20 bluestore.DeferredBatch(0x7f690616b560) prepare_write seq 1322 0x2cc8fc10000~1000 crc 558b6240 2017-09-22 09:46:50.186894 7f6903e8dd00 20 bluestore(/var/lib/ceph/osd/ceph-19/) _deferred_replay replay 0x000000000000052b 2017-09-22 09:46:50.186914 7f6903e8dd00 20 bluestore(/var/lib/ceph/osd/ceph-19/) _txc_create osr 0x7f6906126060 = 0x7f6905fb0b00 seq 2 2017-09-22 09:46:50.186916 7f6903e8dd00 10 bluestore(/var/lib/ceph/osd/ceph-19/) _txc_state_proc txc 0x7f6905fb0b00 kv_done 2017-09-22 09:46:50.186917 7f6903e8dd00 20 bluestore(/var/lib/ceph/osd/ceph-19/) _deferred_queue txc 0x7f6905fb0b00 osr 0x7f6906126060 2017-09-22 09:46:50.186921 7f6903e8dd00 20 bluestore.DeferredBatch(0x7f690616b560) prepare_write seq 1323 0x11a3e4a0000~20000 crc c3b40f52 2017-09-22 09:46:50.187594 7f6903e8dd00 20 bluestore(/var/lib/ceph/osd/ceph-19/) _deferred_replay replay 0x000000000000052c 2017-09-22 09:46:50.187612 7f6903e8dd00 20 bluestore(/var/lib/ceph/osd/ceph-19/) _txc_create osr 0x7f6906126060 = 0x7f6905fb1080 seq 3 2017-09-22 09:46:50.187614 7f6903e8dd00 10 bluestore(/var/lib/ceph/osd/ceph-19/) _txc_state_proc txc 0x7f6905fb1080 kv_done 2017-09-22 09:46:50.187615 7f6903e8dd00 20 bluestore(/var/lib/ceph/osd/ceph-19/) _deferred_queue txc 0x7f6905fb1080 osr 0x7f6906126060 2017-09-22 09:46:50.187617 7f6903e8dd00 20 bluestore.DeferredBatch(0x7f690616b560) prepare_write seq 1324 0x11852080000~20000 crc 968a645d 2017-09-22 09:46:50.188266 7f6903e8dd00 20 bluestore(/var/lib/ceph/osd/ceph-19/) _deferred_replay replay 0x000000000000052d 2017-09-22 09:46:50.188285 7f6903e8dd00 20 bluestore(/var/lib/ceph/osd/ceph-19/) _txc_create osr 0x7f6906126060 = 0x7f6905fb0840 seq 4 2017-09-22 09:46:50.188288 7f6903e8dd00 10 bluestore(/var/lib/ceph/osd/ceph-19/) _txc_state_proc txc 0x7f6905fb0840 kv_done 2017-09-22 09:46:50.188289 7f6903e8dd00 20 bluestore(/var/lib/ceph/osd/ceph-19/) _deferred_queue txc 0x7f6905fb0840 osr 0x7f6906126060 2017-09-22 09:46:50.188297 7f6903e8dd00 20 bluestore.DeferredBatch(0x7f690616b560) prepare_write seq 1325 0x3cc3400000~20000 crc ae53110f 2017-09-22 09:46:50.188562 7f6903e8dd00 20 bluestore(/var/lib/ceph/osd/ceph-19/) _deferred_replay replay 0x000000000000052e 2017-09-22 09:46:50.188581 7f6903e8dd00 20 bluestore(/var/lib/ceph/osd/ceph-19/) _txc_create osr 0x7f6906126060 = 0x7f6905fb0580 seq 5 2017-09-22 09:46:50.188582 7f6903e8dd00 10 bluestore(/var/lib/ceph/osd/ceph-19/) _txc_state_proc txc 0x7f6905fb0580 kv_done 2017-09-22 09:46:50.188583 7f6903e8dd00 20 bluestore(/var/lib/ceph/osd/ceph-19/) _deferred_queue txc 0x7f6905fb0580 osr 0x7f6906126060 2017-09-22 09:46:50.188585 7f6903e8dd00 20 bluestore.DeferredBatch(0x7f690616b560) prepare_write seq 1326 0x2cc91c07000~2000 crc b39fb09b 2017-09-22 09:46:50.188589 7f6903e8dd00 20 bluestore.DeferredBatch(0x7f690616b560) prepare_write seq 1326 0x2cc91c10000~1000 crc cb49bc95 2017-09-22 09:46:50.197357 7f6903e8dd00 20 bluestore(/var/lib/ceph/osd/ceph-19/) _deferred_replay draining osr 2017-09-22 09:46:50.197365 7f6903e8dd00 10 bluestore(/var/lib/ceph/osd/ceph-19/) _osr_drain_all 2017-09-22 09:46:50.197367 7f6903e8dd00 20 bluestore(/var/lib/ceph/osd/ceph-19/) _osr_drain_all osr_set 0x7f6906126060 2017-09-22 09:46:50.197369 7f6903e8dd00 20 bluestore(/var/lib/ceph/osd/ceph-19/) deferred_try_submit 1 osrs, 5 txcs 2017-09-22 09:46:50.197371 7f6903e8dd00 10 bluestore(/var/lib/ceph/osd/ceph-19/) _deferred_submit_unlock osr 0x7f6906126060 7 ios pending 2017-09-22 09:46:50.197375 7f6903e8dd00 20 bluestore(/var/lib/ceph/osd/ceph-19/) _deferred_submit_unlock seq 1325 0x3cc3400000~20000 2017-09-22 09:46:50.197377 7f6903e8dd00 20 bluestore(/var/lib/ceph/osd/ceph-19/) _deferred_submit_unlock write 0x3cc3400000~20000 crc ae53110f 2017-09-22 09:46:50.197459 7f6903e8dd00 20 bluestore(/var/lib/ceph/osd/ceph-19/) _deferred_submit_unlock seq 1324 0x11852080000~20000 2017-09-22 09:46:50.197462 7f6903e8dd00 20 bluestore(/var/lib/ceph/osd/ceph-19/) _deferred_submit_unlock write 0x11852080000~20000 crc 968a645d 2017-09-22 09:46:50.197533 7f6903e8dd00 20 bluestore(/var/lib/ceph/osd/ceph-19/) _deferred_submit_unlock seq 1323 0x11a3e4a0000~20000 2017-09-22 09:46:50.197535 7f6903e8dd00 20 bluestore(/var/lib/ceph/osd/ceph-19/) _deferred_submit_unlock write 0x11a3e4a0000~20000 crc c3b40f52 2017-09-22 09:46:50.197604 7f6903e8dd00 20 bluestore(/var/lib/ceph/osd/ceph-19/) _deferred_submit_unlock seq 1322 0x2cc8fc04000~2000 2017-09-22 09:46:50.197607 7f6903e8dd00 20 bluestore(/var/lib/ceph/osd/ceph-19/) _deferred_submit_unlock write 0x2cc8fc04000~2000 crc c2a954b4 2017-09-22 09:46:50.197610 7f6903e8dd00 20 bluestore(/var/lib/ceph/osd/ceph-19/) _deferred_submit_unlock seq 1322 0x2cc8fc10000~1000 2017-09-22 09:46:50.197612 7f6903e8dd00 20 bluestore(/var/lib/ceph/osd/ceph-19/) _deferred_submit_unlock write 0x2cc8fc10000~1000 crc 558b6240 2017-09-22 09:46:50.197614 7f6903e8dd00 20 bluestore(/var/lib/ceph/osd/ceph-19/) _deferred_submit_unlock seq 1326 0x2cc91c07000~2000 2017-09-22 09:46:50.197615 7f6903e8dd00 20 bluestore(/var/lib/ceph/osd/ceph-19/) _deferred_submit_unlock write 0x2cc91c07000~2000 crc b39fb09b 2017-09-22 09:46:50.197621 7f6903e8dd00 20 bluestore(/var/lib/ceph/osd/ceph-19/) _deferred_submit_unlock seq 1326 0x2cc91c10000~1000 2017-09-22 09:46:50.197623 7f6903e8dd00 20 bluestore(/var/lib/ceph/osd/ceph-19/) _deferred_submit_unlock write 0x2cc91c10000~1000 crc cb49bc95 2017-09-22 09:46:50.197703 7f6903e8dd00 20 bluestore(/var/lib/ceph/osd/ceph-19/) _osr_drain_all drain 0x7f6906126060 2017-09-22 09:46:50.197710 7f68f07d2700 20 bluestore(/var/lib/ceph/osd/ceph-19/) _kv_sync_thread wake 2017-09-22 09:46:50.197710 7f68effd1700 20 bluestore(/var/lib/ceph/osd/ceph-19/) _kv_finalize_thread wake 2017-09-22 09:46:50.197722 7f68f07d2700 20 bluestore(/var/lib/ceph/osd/ceph-19/) _kv_sync_thread sleep 2017-09-22 09:46:50.197724 7f68effd1700 20 bluestore(/var/lib/ceph/osd/ceph-19/) _kv_finalize_thread sleep 2017-09-22 09:46:50.199055 7f68f4fdb700 10 bluestore(/var/lib/ceph/osd/ceph-19/) _deferred_aio_finish osr 0x7f6906126060 2017-09-22 09:46:50.199067 7f68f4fdb700 20 bluestore(/var/lib/ceph/osd/ceph-19/) _deferred_aio_finish dequeueing 2017-09-22 09:46:50.199087 7f68f07d2700 20 bluestore(/var/lib/ceph/osd/ceph-19/) _kv_sync_thread wake 2017-09-22 09:46:50.199095 7f68f07d2700 20 bluestore(/var/lib/ceph/osd/ceph-19/) _kv_sync_thread committing 0 submitting 0 deferred done 1 stable 0 2017-09-22 09:46:50.199100 7f68f07d2700 20 bluestore(/var/lib/ceph/osd/ceph-19/) _kv_sync_thread num_aios=0 force_flush=1, flushing, deferred done->stable 2017-09-22 09:46:50.316440 7f68f07d2700 10 bluestore(/var/lib/ceph/osd/ceph-19/) _kv_sync_thread new_nid_max 713690 2017-09-22 09:46:50.316454 7f68f07d2700 10 bluestore(/var/lib/ceph/osd/ceph-19/) _kv_sync_thread new_blobid_max 174080 2017-09-22 09:46:50.316466 7f68f07d2700 10 bluestore(/var/lib/ceph/osd/ceph-19/) _balance_bluefs_freespace bluefs 148 G free (0.993998) bluestore 856 G free (0.22985), bluefs_ratio 0.147472 2017-09-22 09:46:50.316534 7f68f07d2700 -1 rocksdb: submit_transaction_sync error: Corruption: block checksum mismatch code = 2 Rocksdb transaction: Put( Prefix = S key = 'nid_max' Value size = 8) Put( Prefix = S key = 'blobid_max' Value size = 8) SingleDelete(Prefix = L Key = 0x000000000000052a) SingleDelete(Prefix = L Key = 0x000000000000052b) SingleDelete(Prefix = L Key = 0x000000000000052c) SingleDelete(Prefix = L Key = 0x000000000000052d) SingleDelete(Prefix = L Key = 0x000000000000052e)/home/zstor/ceph/rpmbuild/BUILD/ceph-12.2.0/src/os/bluestore/BlueStore.cc: In function 'void BlueStore::_kv_sync_thread()' thread 7f68f07d2700 time 2017-09-22 09:46:50.316550 /home/zstor/ceph/rpmbuild/BUILD/ceph-12.2.0/src/os/bluestore/BlueStore.cc: 8545: FAILED assert(r == 0) ceph version 12.2.0-3-4-g9a59fd6 (9a59fd686ef3d9445911fae9cdb081c148301a4d) luminous (stable) 1: (ceph::__ceph_assert_fail(char const*, char const*, int, char const*)+0x110) [0x7f68fa57f590] 2: (BlueStore::_kv_sync_thread()+0x35ee) [0x7f69041c147e] 3: (BlueStore::KVSyncThread::entry()+0xd) [0x7f690420982d] 4: (()+0x7df3) [0x7f68f8e42df3] 5: (clone()+0x6d) [0x7f68f79023ed] NOTE: a copy of the executable, or `objdump -rdS ` is needed to interpret this. 2017-09-22 09:46:50.317302 7f68f07d2700 -1 /home/zstor/ceph/rpmbuild/BUILD/ceph-12.2.0/src/os/bluestore/BlueStore.cc: In function 'void BlueStore::_kv_sync_thread()' thread 7f68f07d2700 time 2017-09-22 09:46:50.316550 /home/zstor/ceph/rpmbuild/BUILD/ceph-12.2.0/src/os/bluestore/BlueStore.cc: 8545: FAILED assert(r == 0) ceph version 12.2.0-3-4-g9a59fd6 (9a59fd686ef3d9445911fae9cdb081c148301a4d) luminous (stable) 1: (ceph::__ceph_assert_fail(char const*, char const*, int, char const*)+0x110) [0x7f68fa57f590] 2: (BlueStore::_kv_sync_thread()+0x35ee) [0x7f69041c147e] 3: (BlueStore::KVSyncThread::entry()+0xd) [0x7f690420982d] 4: (()+0x7df3) [0x7f68f8e42df3] 5: (clone()+0x6d) [0x7f68f79023ed] NOTE: a copy of the executable, or `objdump -rdS ` is needed to interpret this. --- begin dump of recent events --- -500> 2017-09-22 09:46:37.941615 7f6903e8dd00 10 bitmapalloc:init_rm_free instance 140089048682960 offset 0xf0000000 length 0x4200000 -499> 2017-09-22 09:46:37.941618 7f6903e8dd00 10 bitmapalloc:init_rm_free instance 140089048682960 offset 0xeb700000 length 0xe00000 -498> 2017-09-22 09:46:37.941619 7f6903e8dd00 10 bitmapalloc:init_rm_free instance 140089048682960 offset 0xecc00000 length 0x3400000 -497> 2017-09-22 09:46:37.941620 7f6903e8dd00 10 bitmapalloc:init_rm_free instance 140089048682960 offset 0xb8900000 length 0x500000 -496> 2017-09-22 09:46:37.941621 7f6903e8dd00 10 bitmapalloc:init_rm_free instance 140089048682960 offset 0xd9800000 length 0x900000 -495> 2017-09-22 09:46:37.941622 7f6903e8dd00 10 bitmapalloc:init_rm_free instance 140089048682960 offset 0xe8300000 length 0x3400000 -494> 2017-09-22 09:46:37.941623 7f6903e8dd00 10 bitmapalloc:init_rm_free instance 140089048682960 offset 0xb4700000 length 0x4200000 -493> 2017-09-22 09:46:37.941624 7f6903e8dd00 10 bitmapalloc:init_rm_free instance 140089048682960 offset 0xb0500000 length 0x1800000 -492> 2017-09-22 09:46:37.941626 7f6903e8dd00 10 bitmapalloc:init_rm_free instance 140089048682960 offset 0xb3800000 length 0xf00000 -491> 2017-09-22 09:46:37.941627 7f6903e8dd00 10 bitmapalloc:init_rm_free instance 140089048682960 offset 0xa5600000 length 0x2d00000 -490> 2017-09-22 09:46:37.941628 7f6903e8dd00 10 bitmapalloc:init_rm_free instance 140089048682960 offset 0xaae00000 length 0x600000 -489> 2017-09-22 09:46:37.941629 7f6903e8dd00 10 bitmapalloc:init_rm_free instance 140089048682960 offset 0xaf600000 length 0xf00000 -488> 2017-09-22 09:46:37.941630 7f6903e8dd00 10 bitmapalloc:init_rm_free instance 140089048682960 offset 0xa1400000 length 0x4200000 -487> 2017-09-22 09:46:37.941631 7f6903e8dd00 10 bitmapalloc:init_rm_free instance 140089048682960 offset 0x74900000 length 0x900000 -486> 2017-09-22 09:46:37.941633 7f6903e8dd00 10 bitmapalloc:init_rm_free instance 140089048682960 offset 0x79100000 length 0x2500000 -485> 2017-09-22 09:46:37.941634 7f6903e8dd00 10 bitmapalloc:init_rm_free instance 140089048682960 offset 0x83700000 length 0x1400000 -484> 2017-09-22 09:46:37.941635 7f6903e8dd00 10 bitmapalloc:init_rm_free instance 140089048682960 offset 0x70700000 length 0x4200000 -483> 2017-09-22 09:46:37.941636 7f6903e8dd00 10 bitmapalloc:init_rm_free instance 140089048682960 offset 0x58500000 length 0x2100000 -482> 2017-09-22 09:46:37.941637 7f6903e8dd00 10 bitmapalloc:init_rm_free instance 140089048682960 offset 0x6a400000 length 0x2100000 -481> 2017-09-22 09:46:37.941638 7f6903e8dd00 10 bitmapalloc:init_rm_free instance 140089048682960 offset 0x50a00000 length 0x2a00000 -480> 2017-09-22 09:46:37.941639 7f6903e8dd00 10 bitmapalloc:init_rm_free instance 140089048682960 offset 0x53500000 length 0x1700000 -479> 2017-09-22 09:46:37.941640 7f6903e8dd00 10 bitmapalloc:init_rm_free instance 140089048682960 offset 0x58400000 length 0x100000 -478> 2017-09-22 09:46:37.941641 7f6903e8dd00 10 bitmapalloc:init_rm_free instance 140089048682960 offset 0x2a500000 length 0x2500000 -477> 2017-09-22 09:46:37.941642 7f6903e8dd00 10 bitmapalloc:init_rm_free instance 140089048682960 offset 0x2fb00000 length 0xa00000 -476> 2017-09-22 09:46:37.941643 7f6903e8dd00 10 bitmapalloc:init_rm_free instance 140089048682960 offset 0x34800000 length 0x1100000 -475> 2017-09-22 09:46:37.941645 7f6903e8dd00 10 bitmapalloc:init_rm_free instance 140089048682960 offset 0x25e00000 length 0x1e00000 -474> 2017-09-22 09:46:37.941646 7f6903e8dd00 10 bitmapalloc:init_rm_free instance 140089048682960 offset 0x28800000 length 0x1d00000 -473> 2017-09-22 09:46:37.941646 7f6903e8dd00 10 bitmapalloc:init_rm_free instance 140089048682960 offset 0xc500000 length 0x4200000 -472> 2017-09-22 09:46:37.941648 7f6903e8dd00 10 bitmapalloc:init_rm_free instance 140089048682960 offset 0x12d500000 length 0x4200000 -471> 2017-09-22 09:46:37.941649 7f6903e8dd00 10 bitmapalloc:init_rm_free instance 140089048682960 offset 0x87d00000 length 0xc00000 -470> 2017-09-22 09:46:37.941651 7f6903e8dd00 10 bitmapalloc:init_rm_free instance 140089048682960 offset 0x80500000 length 0x3200000 -469> 2017-09-22 09:46:37.941651 7f6903e8dd00 10 bitmapalloc:init_rm_free instance 140089048682960 offset 0x86c00000 length 0x1100000 -468> 2017-09-22 09:46:37.941652 7f6903e8dd00 10 bitmapalloc:init_rm_free instance 140089048682960 offset 0x75c00000 length 0x600000 -467> 2017-09-22 09:46:37.941653 7f6903e8dd00 10 bitmapalloc:init_rm_free instance 140089048682960 offset 0x7c900000 length 0x3c00000 -466> 2017-09-22 09:46:37.941654 7f6903e8dd00 10 bitmapalloc:init_rm_free instance 140089048682960 offset 0x66c00000 length 0x3800000 -465> 2017-09-22 09:46:37.941656 7f6903e8dd00 10 bitmapalloc:init_rm_free instance 140089048682960 offset 0x75200000 length 0xa00000 -464> 2017-09-22 09:46:37.941656 7f6903e8dd00 10 bitmapalloc:init_rm_free instance 140089048682960 offset 0x62a00000 length 0x4200000 -463> 2017-09-22 09:46:37.941657 7f6903e8dd00 10 bitmapalloc:init_rm_free instance 140089048682960 offset 0x32200000 length 0x2600000 -462> 2017-09-22 09:46:37.941659 7f6903e8dd00 10 bitmapalloc:init_rm_free instance 140089048682960 offset 0x38a00000 length 0x1c00000 -461> 2017-09-22 09:46:37.941660 7f6903e8dd00 10 bitmapalloc:init_rm_free instance 140089048682960 offset 0x2d600000 length 0x2500000 -460> 2017-09-22 09:46:37.941662 7f6903e8dd00 10 bitmapalloc:init_rm_free instance 140089048682960 offset 0x30500000 length 0x1d00000 -459> 2017-09-22 09:46:37.941663 7f6903e8dd00 10 bitmapalloc:init_rm_free instance 140089048683008 offset 0x1bf74200000 length 0x3300000 -458> 2017-09-22 09:46:37.941664 7f6903e8dd00 10 bitmapalloc:init_rm_free instance 140089048683008 offset 0x1bf66400000 length 0x1400000 -457> 2017-09-22 09:46:37.941665 7f6903e8dd00 10 bitmapalloc:init_rm_free instance 140089048683008 offset 0x1bf6be00000 length 0x1000000 -456> 2017-09-22 09:46:37.941666 7f6903e8dd00 10 bitmapalloc:init_rm_free instance 140089048683008 offset 0x1bf72400000 length 0x1e00000 -455> 2017-09-22 09:46:37.941667 7f6903e8dd00 10 bitmapalloc:init_rm_free instance 140089048682960 offset 0x14900000 length 0x100000 -454> 2017-09-22 09:46:37.941668 7f6903e8dd00 10 bitmapalloc:init_rm_free instance 140089048682960 offset 0x15700000 length 0x1700000 -453> 2017-09-22 09:46:37.941669 7f6903e8dd00 10 bitmapalloc:init_rm_free instance 140089048682960 offset 0x1b000000 length 0x2700000 -452> 2017-09-22 09:46:37.941670 7f6903e8dd00 10 bitmapalloc:init_rm_free instance 140089048682960 offset 0x25b00000 length 0x300000 -451> 2017-09-22 09:46:37.941671 7f6903e8dd00 10 bitmapalloc:init_rm_free instance 140089048683008 offset 0x1bf62200000 length 0x4200000 -450> 2017-09-22 09:46:37.941672 7f6903e8dd00 10 bitmapalloc:init_rm_free instance 140089048683008 offset 0x1bf59b00000 length 0x1300000 -449> 2017-09-22 09:46:37.941673 7f6903e8dd00 10 bitmapalloc:init_rm_free instance 140089048683008 offset 0x1bf5f300000 length 0x2f00000 -448> 2017-09-22 09:46:37.941674 7f6903e8dd00 10 bitmapalloc:init_rm_free instance 140089048683008 offset 0x1bf34d00000 length 0x2500000 -447> 2017-09-22 09:46:37.941675 7f6903e8dd00 10 bitmapalloc:init_rm_free instance 140089048683008 offset 0x1bf3c100000 length 0x1d00000 -446> 2017-09-22 09:46:37.941676 7f6903e8dd00 10 bitmapalloc:init_rm_free instance 140089048682960 offset 0x3a600000 length 0x4200000 -445> 2017-09-22 09:46:37.941677 7f6903e8dd00 10 bitmapalloc:init_rm_free instance 140089048682960 offset 0x6c500000 length 0x4200000 -444> 2017-09-22 09:46:37.941679 7f6903e8dd00 10 bitmapalloc:init_rm_free instance 140089048682960 offset 0xab400000 length 0x4200000 -443> 2017-09-22 09:46:37.941680 7f6903e8dd00 10 bitmapalloc:init_rm_free instance 140089048682960 offset 0x4700000 length 0x100000 -442> 2017-09-22 09:46:37.941681 7f6903e8dd00 10 bitmapalloc:init_rm_free instance 140089048682864 offset 0x30100000 length 0x8800000 -441> 2017-09-22 09:46:37.941682 7f6903e8dd00 10 bitmapalloc:init_rm_free instance 140089048682960 offset 0x84b00000 length 0x2100000 -440> 2017-09-22 09:46:37.941684 7f6903e8dd00 10 bitmapalloc:init_rm_free instance 140089048682960 offset 0x8e100000 length 0x500000 -439> 2017-09-22 09:46:37.941684 7f6903e8dd00 10 bitmapalloc:init_rm_free instance 140089048682960 offset 0x8f000000 length 0x900000 -438> 2017-09-22 09:46:37.941685 7f6903e8dd00 10 bitmapalloc:init_rm_free instance 140089048682960 offset 0xa0100000 length 0x1300000 -437> 2017-09-22 09:46:37.941686 7f6903e8dd00 10 bitmapalloc:init_rm_free instance 140089048682960 offset 0x76200000 length 0x2f00000 -436> 2017-09-22 09:46:37.941687 7f6903e8dd00 10 bitmapalloc:init_rm_free instance 140089048682960 offset 0x7b600000 length 0x1300000 -435> 2017-09-22 09:46:37.941688 7f6903e8dd00 10 bitmapalloc:init_rm_free instance 140089048682960 offset 0x93b00000 length 0x4200000 -434> 2017-09-22 09:46:37.941693 7f6903e8dd00 10 bitmapalloc:init_rm_free instance 140089048682960 offset 0x8f900000 length 0x4200000 -433> 2017-09-22 09:46:37.941694 7f6903e8dd00 10 bitmapalloc:init_rm_free instance 140089048682960 offset 0x2b00000 length 0x100000 -432> 2017-09-22 09:46:37.941701 7f6903e8dd00 10 bitmapalloc:init_rm_free instance 140089048683008 offset 0x1bf4d600000 length 0x4200000 -431> 2017-09-22 09:46:37.941702 7f6903e8dd00 10 bitmapalloc:init_rm_free instance 140089048682960 offset 0x42a00000 length 0x4200000 -430> 2017-09-22 09:46:37.941703 7f6903e8dd00 10 bitmapalloc:init_rm_free instance 140089048682960 offset 0x35900000 length 0x3100000 -429> 2017-09-22 09:46:37.941704 7f6903e8dd00 10 bitmapalloc:init_rm_free instance 140089048682960 offset 0x4d000000 length 0x1100000 -428> 2017-09-22 09:46:37.941705 7f6903e8dd00 10 bitmapalloc:init_rm_free instance 140089048682960 offset 0x300000 length 0x100000 -427> 2017-09-22 09:46:37.941706 7f6903e8dd00 10 bitmapalloc:init_rm_free instance 140089048682960 offset 0x50100000 length 0x900000 -426> 2017-09-22 09:46:37.941708 7f6903e8dd00 10 bitmapalloc:init_rm_free instance 140089048682960 offset 0x53400000 length 0x100000 -425> 2017-09-22 09:46:37.941708 7f6903e8dd00 10 bitmapalloc:init_rm_free instance 140089048682960 offset 0x54c00000 length 0x3800000 -424> 2017-09-22 09:46:37.941709 7f6903e8dd00 10 bitmapalloc:init_rm_free instance 140089048682960 offset 0x134c00000 length 0x1200000 -423> 2017-09-22 09:46:37.941714 7f6903e8dd00 10 bitmapalloc:init_rm_free instance 140089048682960 offset 0xe1e00000 length 0x4200000 -422> 2017-09-22 09:46:37.941715 7f6903e8dd00 10 bitmapalloc:init_rm_free instance 140089048683008 offset 0x1bf42000000 length 0x3e00000 -421> 2017-09-22 09:46:37.941716 7f6903e8dd00 10 bitmapalloc:init_rm_free instance 140089048683008 offset 0x1bf48400000 length 0x400000 -420> 2017-09-22 09:46:37.941717 7f6903e8dd00 10 bitmapalloc:init_rm_free instance 140089048682960 offset 0x3e800000 length 0x4200000 -419> 2017-09-22 09:46:37.941718 7f6903e8dd00 10 bitmapalloc:init_rm_free instance 140089048682960 offset 0xda100000 length 0x2400000 -418> 2017-09-22 09:46:37.941719 7f6903e8dd00 10 bitmapalloc:init_rm_free instance 140089048682960 offset 0xe0000000 length 0x1e00000 -417> 2017-09-22 09:46:37.941720 7f6903e8dd00 10 bitmapalloc:init_rm_free instance 140089048683008 offset 0x1bf3de00000 length 0x4200000 -416> 2017-09-22 09:46:37.941721 7f6903e8dd00 10 bitmapalloc:init_rm_free instance 140089048682960 offset 0x46c00000 length 0x4200000 -415> 2017-09-22 09:46:37.941722 7f6903e8dd00 10 bitmapalloc:init_rm_free instance 140089048682960 offset 0x5a600000 length 0x4200000 -414> 2017-09-22 09:46:37.941723 7f6903e8dd00 10 bitmapalloc:init_rm_free instance 140089048682960 offset 0xcef00000 length 0x2e00000 -413> 2017-09-22 09:46:37.941724 7f6903e8dd00 10 bitmapalloc:init_rm_free instance 140089048682960 offset 0xd4100000 length 0x1400000 -412> 2017-09-22 09:46:37.941725 7f6903e8dd00 10 bitmapalloc:init_rm_free instance 140089048682960 offset 0x4ae00000 length 0x2200000 -411> 2017-09-22 09:46:37.941726 7f6903e8dd00 10 bitmapalloc:init_rm_free instance 140089048682960 offset 0x4e100000 length 0x2000000 -410> 2017-09-22 09:46:37.941727 7f6903e8dd00 10 bitmapalloc:init_rm_free instance 140089048683008 offset 0x1bf20c00000 length 0x4600000 -409> 2017-09-22 09:46:37.941728 7f6903e8dd00 10 bitmapalloc:init_rm_free instance 140089048683008 offset 0x1bf1c600000 length 0x4600000 -408> 2017-09-22 09:46:37.941728 7f6903e8dd00 10 bitmapalloc:init_rm_free instance 140089048682960 offset 0xd5500000 length 0xb00000 -407> 2017-09-22 09:46:37.941729 7f6903e8dd00 10 bitmapalloc:init_rm_free instance 140089048682960 offset 0x5e800000 length 0x4200000 -406> 2017-09-22 09:46:37.941731 7f6903e8dd00 10 bitmapalloc:init_rm_free instance 140089048683008 offset 0x1bf48800000 length 0x4200000 -405> 2017-09-22 09:46:37.941732 7f6903e8dd00 10 bitmapalloc:init_rm_free instance 140089048682960 offset 0xe6000000 length 0x2300000 -404> 2017-09-22 09:46:37.941733 7f6903e8dd00 10 bitmapalloc:init_rm_free instance 140089048682960 offset 0xec500000 length 0x700000 -403> 2017-09-22 09:46:37.941734 7f6903e8dd00 10 bitmapalloc:init_rm_free instance 140089048683008 offset 0x1bf25200000 length 0x4200000 -402> 2017-09-22 09:46:37.941735 7f6903e8dd00 10 bitmapalloc:init_rm_free instance 140089048683008 offset 0x1bf30b00000 length 0x4200000 -401> 2017-09-22 09:46:37.941782 7f6903e8dd00 10 bluestore(/var/lib/ceph/osd/ceph-19/) _open_db set rocksdb_db_paths to db,38762079846 db.slow,3800647040409 -400> 2017-09-22 09:46:37.941846 7f6903e8dd00 0 set rocksdb option compaction_readahead_size = 2097152 -399> 2017-09-22 09:46:37.941854 7f6903e8dd00 0 set rocksdb option compression = kNoCompression -398> 2017-09-22 09:46:37.941859 7f6903e8dd00 0 set rocksdb option max_write_buffer_number = 4 -397> 2017-09-22 09:46:37.941861 7f6903e8dd00 0 set rocksdb option min_write_buffer_number_to_merge = 1 -396> 2017-09-22 09:46:37.941870 7f6903e8dd00 0 set rocksdb option recycle_log_file_num = 4 -395> 2017-09-22 09:46:37.941872 7f6903e8dd00 0 set rocksdb option writable_file_max_buffer_size = 0 -394> 2017-09-22 09:46:37.941881 7f6903e8dd00 0 set rocksdb option write_buffer_size = 268435456 -393> 2017-09-22 09:46:37.941902 7f6903e8dd00 0 set rocksdb option compaction_readahead_size = 2097152 -392> 2017-09-22 09:46:37.941904 7f6903e8dd00 0 set rocksdb option compression = kNoCompression -391> 2017-09-22 09:46:37.941907 7f6903e8dd00 0 set rocksdb option max_write_buffer_number = 4 -390> 2017-09-22 09:46:37.941909 7f6903e8dd00 0 set rocksdb option min_write_buffer_number_to_merge = 1 -389> 2017-09-22 09:46:37.941911 7f6903e8dd00 0 set rocksdb option recycle_log_file_num = 4 -388> 2017-09-22 09:46:37.941912 7f6903e8dd00 0 set rocksdb option writable_file_max_buffer_size = 0 -387> 2017-09-22 09:46:37.941915 7f6903e8dd00 0 set rocksdb option write_buffer_size = 268435456 -386> 2017-09-22 09:46:37.942042 7f6903e8dd00 4 rocksdb: RocksDB version: 5.4.0 -385> 2017-09-22 09:46:37.942047 7f6903e8dd00 4 rocksdb: Git sha rocksdb_build_git_sha:@0@ -384> 2017-09-22 09:46:37.942048 7f6903e8dd00 4 rocksdb: Compile date Sep 21 2017 -383> 2017-09-22 09:46:37.942051 7f6903e8dd00 4 rocksdb: DB SUMMARY -382> 2017-09-22 09:46:37.942078 7f6903e8dd00 4 rocksdb: CURRENT file: CURRENT -381> 2017-09-22 09:46:37.942079 7f6903e8dd00 4 rocksdb: IDENTITY file: IDENTITY -380> 2017-09-22 09:46:37.942084 7f6903e8dd00 4 rocksdb: MANIFEST file: MANIFEST-001678 size: 10657 Bytes -379> 2017-09-22 09:46:37.942086 7f6903e8dd00 4 rocksdb: SST files in db dir, Total Num: 49, files: 001173.sst 001235.sst 001236.sst 001296.sst 001297.sst 001298.sst 001424.sst 001425.sst 001469.sst -378> 2017-09-22 09:46:37.942092 7f6903e8dd00 4 rocksdb: SST files in db.slow dir, Total Num: 14, files: 000628.sst 000680.sst 001040.sst 001490.sst 001491.sst 001492.sst 001493.sst 001494.sst 001495.sst -377> 2017-09-22 09:46:37.942099 7f6903e8dd00 4 rocksdb: Write Ahead Log file in db.wal: 001679.log size: 0 ; -376> 2017-09-22 09:46:37.942101 7f6903e8dd00 4 rocksdb: Options.error_if_exists: 0 -375> 2017-09-22 09:46:37.942102 7f6903e8dd00 4 rocksdb: Options.create_if_missing: 0 -374> 2017-09-22 09:46:37.942103 7f6903e8dd00 4 rocksdb: Options.paranoid_checks: 1 -373> 2017-09-22 09:46:37.942104 7f6903e8dd00 4 rocksdb: Options.env: 0x7f6905f784c0 -372> 2017-09-22 09:46:37.942106 7f6903e8dd00 4 rocksdb: Options.info_log: 0x7f6905f78da0 -371> 2017-09-22 09:46:37.942107 7f6903e8dd00 4 rocksdb: Options.max_open_files: -1 -370> 2017-09-22 09:46:37.942107 7f6903e8dd00 4 rocksdb: Options.max_file_opening_threads: 16 -369> 2017-09-22 09:46:37.942108 7f6903e8dd00 4 rocksdb: Options.use_fsync: 0 -368> 2017-09-22 09:46:37.942109 7f6903e8dd00 4 rocksdb: Options.max_log_file_size: 0 -367> 2017-09-22 09:46:37.942110 7f6903e8dd00 4 rocksdb: Options.max_manifest_file_size: 18446744073709551615 -366> 2017-09-22 09:46:37.942111 7f6903e8dd00 4 rocksdb: Options.log_file_time_to_roll: 0 -365> 2017-09-22 09:46:37.942111 7f6903e8dd00 4 rocksdb: Options.keep_log_file_num: 1000 -364> 2017-09-22 09:46:37.942112 7f6903e8dd00 4 rocksdb: Options.recycle_log_file_num: 4 -363> 2017-09-22 09:46:37.942113 7f6903e8dd00 4 rocksdb: Options.allow_fallocate: 1 -362> 2017-09-22 09:46:37.942113 7f6903e8dd00 4 rocksdb: Options.allow_mmap_reads: 0 -361> 2017-09-22 09:46:37.942114 7f6903e8dd00 4 rocksdb: Options.allow_mmap_writes: 0 -360> 2017-09-22 09:46:37.942114 7f6903e8dd00 4 rocksdb: Options.use_direct_reads: 0 -359> 2017-09-22 09:46:37.942115 7f6903e8dd00 4 rocksdb: Options.use_direct_io_for_flush_and_compaction: 0 -358> 2017-09-22 09:46:37.942116 7f6903e8dd00 4 rocksdb: Options.create_missing_column_families: 0 -357> 2017-09-22 09:46:37.942117 7f6903e8dd00 4 rocksdb: Options.db_log_dir: -356> 2017-09-22 09:46:37.942117 7f6903e8dd00 4 rocksdb: Options.wal_dir: db.wal -355> 2017-09-22 09:46:37.942118 7f6903e8dd00 4 rocksdb: Options.table_cache_numshardbits: 6 -354> 2017-09-22 09:46:37.942119 7f6903e8dd00 4 rocksdb: Options.max_subcompactions: 1 -353> 2017-09-22 09:46:37.942119 7f6903e8dd00 4 rocksdb: Options.max_background_flushes: 1 -352> 2017-09-22 09:46:37.942120 7f6903e8dd00 4 rocksdb: Options.WAL_ttl_seconds: 0 -351> 2017-09-22 09:46:37.942121 7f6903e8dd00 4 rocksdb: Options.WAL_size_limit_MB: 0 -350> 2017-09-22 09:46:37.942122 7f6903e8dd00 4 rocksdb: Options.manifest_preallocation_size: 4194304 -349> 2017-09-22 09:46:37.942122 7f6903e8dd00 4 rocksdb: Options.is_fd_close_on_exec: 1 -348> 2017-09-22 09:46:37.942123 7f6903e8dd00 4 rocksdb: Options.advise_random_on_open: 1 -347> 2017-09-22 09:46:37.942124 7f6903e8dd00 4 rocksdb: Options.db_write_buffer_size: 0 -346> 2017-09-22 09:46:37.942124 7f6903e8dd00 4 rocksdb: Options.access_hint_on_compaction_start: 1 -345> 2017-09-22 09:46:37.942125 7f6903e8dd00 4 rocksdb: Options.new_table_reader_for_compaction_inputs: 1 -344> 2017-09-22 09:46:37.942125 7f6903e8dd00 4 rocksdb: Options.compaction_readahead_size: 2097152 -343> 2017-09-22 09:46:37.942126 7f6903e8dd00 4 rocksdb: Options.random_access_max_buffer_size: 1048576 -342> 2017-09-22 09:46:37.942127 7f6903e8dd00 4 rocksdb: Options.writable_file_max_buffer_size: 0 -341> 2017-09-22 09:46:37.942127 7f6903e8dd00 4 rocksdb: Options.use_adaptive_mutex: 0 -340> 2017-09-22 09:46:37.942128 7f6903e8dd00 4 rocksdb: Options.rate_limiter: (nil) -339> 2017-09-22 09:46:37.942129 7f6903e8dd00 4 rocksdb: Options.sst_file_manager.rate_bytes_per_sec: 0 -338> 2017-09-22 09:46:37.942130 7f6903e8dd00 4 rocksdb: Options.bytes_per_sync: 0 -337> 2017-09-22 09:46:37.942131 7f6903e8dd00 4 rocksdb: Options.wal_bytes_per_sync: 0 -336> 2017-09-22 09:46:37.942131 7f6903e8dd00 4 rocksdb: Options.wal_recovery_mode: 0 -335> 2017-09-22 09:46:37.942132 7f6903e8dd00 4 rocksdb: Options.enable_thread_tracking: 0 -334> 2017-09-22 09:46:37.942132 7f6903e8dd00 4 rocksdb: Options.allow_concurrent_memtable_write: 1 -333> 2017-09-22 09:46:37.942133 7f6903e8dd00 4 rocksdb: Options.enable_write_thread_adaptive_yield: 1 -332> 2017-09-22 09:46:37.942134 7f6903e8dd00 4 rocksdb: Options.write_thread_max_yield_usec: 100 -331> 2017-09-22 09:46:37.942134 7f6903e8dd00 4 rocksdb: Options.write_thread_slow_yield_usec: 3 -330> 2017-09-22 09:46:37.942135 7f6903e8dd00 4 rocksdb: Options.row_cache: None -329> 2017-09-22 09:46:37.942136 7f6903e8dd00 4 rocksdb: Options.wal_filter: None -328> 2017-09-22 09:46:37.942136 7f6903e8dd00 4 rocksdb: Options.avoid_flush_during_recovery: 0 -327> 2017-09-22 09:46:37.942137 7f6903e8dd00 4 rocksdb: Options.base_background_compactions: 1 -326> 2017-09-22 09:46:37.942138 7f6903e8dd00 4 rocksdb: Options.max_background_compactions: 1 -325> 2017-09-22 09:46:37.942139 7f6903e8dd00 4 rocksdb: Options.avoid_flush_during_shutdown: 0 -324> 2017-09-22 09:46:37.942139 7f6903e8dd00 4 rocksdb: Options.delayed_write_rate : 16777216 -323> 2017-09-22 09:46:37.942140 7f6903e8dd00 4 rocksdb: Options.max_total_wal_size: 0 -322> 2017-09-22 09:46:37.942147 7f6903e8dd00 4 rocksdb: Options.delete_obsolete_files_period_micros: 21600000000 -321> 2017-09-22 09:46:37.942149 7f6903e8dd00 4 rocksdb: Options.stats_dump_period_sec: 600 -320> 2017-09-22 09:46:37.942150 7f6903e8dd00 4 rocksdb: Compression algorithms supported: -319> 2017-09-22 09:46:37.942152 7f6903e8dd00 4 rocksdb: Snappy supported: 0 -318> 2017-09-22 09:46:37.942153 7f6903e8dd00 4 rocksdb: Zlib supported: 0 -317> 2017-09-22 09:46:37.942153 7f6903e8dd00 4 rocksdb: Bzip supported: 0 -316> 2017-09-22 09:46:37.942154 7f6903e8dd00 4 rocksdb: LZ4 supported: 0 -315> 2017-09-22 09:46:37.942154 7f6903e8dd00 4 rocksdb: ZSTD supported: 0 -314> 2017-09-22 09:46:37.942155 7f6903e8dd00 4 rocksdb: Fast CRC32 supported: 1 -313> 2017-09-22 09:46:37.942377 7f6903e8dd00 4 rocksdb: [/home/zstor/ceph/rpmbuild/BUILD/ceph-12.2.0/src/rocksdb/db/version_set.cc:2609] Recovering from manifest file: MANIFEST-001678 -312> 2017-09-22 09:46:37.942435 7f6903e8dd00 4 rocksdb: [/home/zstor/ceph/rpmbuild/BUILD/ceph-12.2.0/src/rocksdb/db/column_family.cc:407] --------------- Options for column family [default]: -311> 2017-09-22 09:46:37.942439 7f6903e8dd00 4 rocksdb: Options.comparator: leveldb.BytewiseComparator -310> 2017-09-22 09:46:37.942445 7f6903e8dd00 4 rocksdb: Options.merge_operator: .T:int64_array.b:bitwise_xor -309> 2017-09-22 09:46:37.942446 7f6903e8dd00 4 rocksdb: Options.compaction_filter: None -308> 2017-09-22 09:46:37.942447 7f6903e8dd00 4 rocksdb: Options.compaction_filter_factory: None -307> 2017-09-22 09:46:37.942447 7f6903e8dd00 4 rocksdb: Options.memtable_factory: SkipListFactory -306> 2017-09-22 09:46:37.942448 7f6903e8dd00 4 rocksdb: Options.table_factory: BlockBasedTable -305> 2017-09-22 09:46:37.942465 7f6903e8dd00 4 rocksdb: table_factory options: flush_block_policy_factory: FlushBlockBySizePolicyFactory (0x7f6905cd2330) cache_index_and_filter_blocks: 1 cache_index_and_filter_blocks_with_high_priority: 1 pin_l0_filter_and_index_blocks_in_cache: 1 index_type: 0 hash_index_allow_collision: 1 checksum: 1 no_block_cache: 0 block_cache: 0x7f6905d17cf8 block_cache_name: LRUCache block_cache_options: capacity : 536870912 num_shard_bits : 4 strict_capacity_limit : 0 high_pri_pool_ratio: 0.000 block_cache_compressed: (nil) persistent_cache: (nil) block_size: 4096 block_size_deviation: 10 block_restart_interval: 16 index_block_restart_interval: 1 filter_policy: rocksdb.BuiltinBloomFilter whole_key_filtering: 1 format_version: 2 -304> 2017-09-22 09:46:37.942475 7f6903e8dd00 4 rocksdb: Options.write_buffer_size: 268435456 -303> 2017-09-22 09:46:37.942476 7f6903e8dd00 4 rocksdb: Options.max_write_buffer_number: 4 -302> 2017-09-22 09:46:37.942477 7f6903e8dd00 4 rocksdb: Options.compression: NoCompression -301> 2017-09-22 09:46:37.942478 7f6903e8dd00 4 rocksdb: Options.bottommost_compression: Disabled -300> 2017-09-22 09:46:37.942479 7f6903e8dd00 4 rocksdb: Options.prefix_extractor: nullptr -299> 2017-09-22 09:46:37.942479 7f6903e8dd00 4 rocksdb: Options.memtable_insert_with_hint_prefix_extractor: nullptr -298> 2017-09-22 09:46:37.942480 7f6903e8dd00 4 rocksdb: Options.num_levels: 7 -297> 2017-09-22 09:46:37.942481 7f6903e8dd00 4 rocksdb: Options.min_write_buffer_number_to_merge: 1 -296> 2017-09-22 09:46:37.942482 7f6903e8dd00 4 rocksdb: Options.max_write_buffer_number_to_maintain: 0 -295> 2017-09-22 09:46:37.942482 7f6903e8dd00 4 rocksdb: Options.compression_opts.window_bits: -14 -294> 2017-09-22 09:46:37.942483 7f6903e8dd00 4 rocksdb: Options.compression_opts.level: -1 -293> 2017-09-22 09:46:37.942484 7f6903e8dd00 4 rocksdb: Options.compression_opts.strategy: 0 -292> 2017-09-22 09:46:37.942484 7f6903e8dd00 4 rocksdb: Options.compression_opts.max_dict_bytes: 0 -291> 2017-09-22 09:46:37.942485 7f6903e8dd00 4 rocksdb: Options.level0_file_num_compaction_trigger: 4 -290> 2017-09-22 09:46:37.942486 7f6903e8dd00 4 rocksdb: Options.level0_slowdown_writes_trigger: 20 -289> 2017-09-22 09:46:37.942492 7f6903e8dd00 4 rocksdb: Options.level0_stop_writes_trigger: 36 -288> 2017-09-22 09:46:37.942492 7f6903e8dd00 4 rocksdb: Options.target_file_size_base: 67108864 -287> 2017-09-22 09:46:37.942493 7f6903e8dd00 4 rocksdb: Options.target_file_size_multiplier: 1 -286> 2017-09-22 09:46:37.942494 7f6903e8dd00 4 rocksdb: Options.max_bytes_for_level_base: 268435456 -285> 2017-09-22 09:46:37.942494 7f6903e8dd00 4 rocksdb: Options.level_compaction_dynamic_level_bytes: 0 -284> 2017-09-22 09:46:37.942495 7f6903e8dd00 4 rocksdb: Options.max_bytes_for_level_multiplier: 10.000000 -283> 2017-09-22 09:46:37.942498 7f6903e8dd00 4 rocksdb: Options.max_bytes_for_level_multiplier_addtl[0]: 1 -282> 2017-09-22 09:46:37.942499 7f6903e8dd00 4 rocksdb: Options.max_bytes_for_level_multiplier_addtl[1]: 1 -281> 2017-09-22 09:46:37.942499 7f6903e8dd00 4 rocksdb: Options.max_bytes_for_level_multiplier_addtl[2]: 1 -280> 2017-09-22 09:46:37.942500 7f6903e8dd00 4 rocksdb: Options.max_bytes_for_level_multiplier_addtl[3]: 1 -279> 2017-09-22 09:46:37.942501 7f6903e8dd00 4 rocksdb: Options.max_bytes_for_level_multiplier_addtl[4]: 1 -278> 2017-09-22 09:46:37.942501 7f6903e8dd00 4 rocksdb: Options.max_bytes_for_level_multiplier_addtl[5]: 1 -277> 2017-09-22 09:46:37.942502 7f6903e8dd00 4 rocksdb: Options.max_bytes_for_level_multiplier_addtl[6]: 1 -276> 2017-09-22 09:46:37.942503 7f6903e8dd00 4 rocksdb: Options.max_sequential_skip_in_iterations: 8 -275> 2017-09-22 09:46:37.942504 7f6903e8dd00 4 rocksdb: Options.max_compaction_bytes: 1677721600 -274> 2017-09-22 09:46:37.942504 7f6903e8dd00 4 rocksdb: Options.arena_block_size: 33554432 -273> 2017-09-22 09:46:37.942505 7f6903e8dd00 4 rocksdb: Options.soft_pending_compaction_bytes_limit: 68719476736 -272> 2017-09-22 09:46:37.942506 7f6903e8dd00 4 rocksdb: Options.hard_pending_compaction_bytes_limit: 274877906944 -271> 2017-09-22 09:46:37.942506 7f6903e8dd00 4 rocksdb: Options.rate_limit_delay_max_milliseconds: 100 -270> 2017-09-22 09:46:37.942507 7f6903e8dd00 4 rocksdb: Options.disable_auto_compactions: 0 -269> 2017-09-22 09:46:37.942508 7f6903e8dd00 4 rocksdb: Options.compaction_style: kCompactionStyleLevel -268> 2017-09-22 09:46:37.942509 7f6903e8dd00 4 rocksdb: Options.compaction_pri: kByCompensatedSize -267> 2017-09-22 09:46:37.942510 7f6903e8dd00 4 rocksdb: Options.compaction_options_universal.size_ratio: 1 -266> 2017-09-22 09:46:37.942511 7f6903e8dd00 4 rocksdb: Options.compaction_options_universal.min_merge_width: 2 -265> 2017-09-22 09:46:37.942511 7f6903e8dd00 4 rocksdb: Options.compaction_options_universal.max_merge_width: 4294967295 -264> 2017-09-22 09:46:37.942513 7f6903e8dd00 4 rocksdb: Options.compaction_options_universal.max_size_amplification_percent: 200 -263> 2017-09-22 09:46:37.942514 7f6903e8dd00 4 rocksdb: Options.compaction_options_universal.compression_size_percent: -1 -262> 2017-09-22 09:46:37.942514 7f6903e8dd00 4 rocksdb: Options.compaction_options_fifo.max_table_files_size: 1073741824 -261> 2017-09-22 09:46:37.942515 7f6903e8dd00 4 rocksdb: Options.table_properties_collectors: -260> 2017-09-22 09:46:37.942516 7f6903e8dd00 4 rocksdb: Options.inplace_update_support: 0 -259> 2017-09-22 09:46:37.942517 7f6903e8dd00 4 rocksdb: Options.inplace_update_num_locks: 10000 -258> 2017-09-22 09:46:37.942518 7f6903e8dd00 4 rocksdb: Options.memtable_prefix_bloom_size_ratio: 0.000000 -257> 2017-09-22 09:46:37.942519 7f6903e8dd00 4 rocksdb: Options.memtable_huge_page_size: 0 -256> 2017-09-22 09:46:37.942519 7f6903e8dd00 4 rocksdb: Options.bloom_locality: 0 -255> 2017-09-22 09:46:37.942520 7f6903e8dd00 4 rocksdb: Options.max_successive_merges: 0 -254> 2017-09-22 09:46:37.942520 7f6903e8dd00 4 rocksdb: Options.optimize_filters_for_hits: 0 -253> 2017-09-22 09:46:37.942521 7f6903e8dd00 4 rocksdb: Options.paranoid_file_checks: 0 -252> 2017-09-22 09:46:37.942522 7f6903e8dd00 4 rocksdb: Options.force_consistency_checks: 0 -251> 2017-09-22 09:46:37.942523 7f6903e8dd00 4 rocksdb: Options.report_bg_io_stats: 0 -250> 2017-09-22 09:46:37.949614 7f6903e8dd00 4 rocksdb: [/home/zstor/ceph/rpmbuild/BUILD/ceph-12.2.0/src/rocksdb/db/version_set.cc:2859] Recovered from manifest file:db/MANIFEST-001678 succeeded,manifest_file_number is 1678, next_file_number is 1680, last_sequence is 19158440, log_number is 0,prev_log_number is 0,max_column_family is 0 -249> 2017-09-22 09:46:37.949625 7f6903e8dd00 4 rocksdb: [/home/zstor/ceph/rpmbuild/BUILD/ceph-12.2.0/src/rocksdb/db/version_set.cc:2867] Column family [default] (ID 0), log number is 1677 -248> 2017-09-22 09:46:37.949734 7f6903e8dd00 4 rocksdb: EVENT_LOG_v1 {"time_micros": 1506044797949726, "job": 1, "event": "recovery_started", "log_files": [1679]} -247> 2017-09-22 09:46:37.949739 7f6903e8dd00 4 rocksdb: [/home/zstor/ceph/rpmbuild/BUILD/ceph-12.2.0/src/rocksdb/db/db_impl_open.cc:482] Recovering log #1679 mode 0 -246> 2017-09-22 09:46:37.949751 7f6903e8dd00 4 rocksdb: [/home/zstor/ceph/rpmbuild/BUILD/ceph-12.2.0/src/rocksdb/db/version_set.cc:2395] Creating manifest 1681 -245> 2017-09-22 09:46:37.949843 7f6903e8dd00 10 bitmapalloc:reserve instance 140089048682960 num_used 2940 total 38912 -244> 2017-09-22 09:46:37.949848 7f6903e8dd00 10 bitmapalloc:allocate instance 140089048682960 want_size 1048576 alloc_unit 1048576 hint 0 -243> 2017-09-22 09:46:37.950197 7f6903e8dd00 10 bitmapalloc:reserve instance 140089048682864 num_used 137 total 6144 -242> 2017-09-22 09:46:37.950204 7f6903e8dd00 10 bitmapalloc:allocate instance 140089048682864 want_size 4194304 alloc_unit 1048576 hint 948961280 -241> 2017-09-22 09:46:37.950702 7f6903e8dd00 10 bitmapalloc:reserve instance 140089048682960 num_used 2941 total 38912 -240> 2017-09-22 09:46:37.950709 7f6903e8dd00 10 bitmapalloc:allocate instance 140089048682960 want_size 1048576 alloc_unit 1048576 hint 0 -239> 2017-09-22 09:46:37.951339 7f6903e8dd00 4 rocksdb: EVENT_LOG_v1 {"time_micros": 1506044797951336, "job": 1, "event": "recovery_finished"} -238> 2017-09-22 09:46:37.951420 7f6903e8dd00 5 rocksdb: [/home/zstor/ceph/rpmbuild/BUILD/ceph-12.2.0/src/rocksdb/db/db_impl_files.cc:307] [JOB 2] Delete db.wal/001679.log type=0 #1679 -- OK -237> 2017-09-22 09:46:37.951429 7f6903e8dd00 5 rocksdb: [/home/zstor/ceph/rpmbuild/BUILD/ceph-12.2.0/src/rocksdb/db/db_impl_files.cc:307] [JOB 2] Delete db//MANIFEST-001678 type=3 #1678 -- OK -236> 2017-09-22 09:46:37.951436 7f6903e8dd00 5 rocksdb: [/home/zstor/ceph/rpmbuild/BUILD/ceph-12.2.0/src/rocksdb/db/db_impl_files.cc:307] [JOB 2] Delete db.slow/001682.sst type=2 #1682 -- OK -235> 2017-09-22 09:46:37.951444 7f6903e8dd00 4 rocksdb: EVENT_LOG_v1 {"time_micros": 1506044797951441, "job": 2, "event": "table_file_deletion", "file_number": 1682} -234> 2017-09-22 09:46:37.951716 7f6903e8dd00 10 bitmapalloc:release 0x2b00000~200000 -233> 2017-09-22 09:46:37.951721 7f6903e8dd00 20 bitmapalloc:insert_free instance 140089048682960 off 0x2b00000 len 0x200000 -232> 2017-09-22 09:46:37.951723 7f6903e8dd00 10 bitmapalloc:release 0x1bf29400000~4100000 -231> 2017-09-22 09:46:37.951724 7f6903e8dd00 20 bitmapalloc:insert_free instance 140089048683008 off 0x1bf29400000 len 0x4100000 -230> 2017-09-22 09:46:37.951831 7f6903e8dd00 10 bitmapalloc:reserve instance 140089048682960 num_used 2940 total 38912 -229> 2017-09-22 09:46:37.951835 7f6903e8dd00 10 bitmapalloc:allocate instance 140089048682960 want_size 1048576 alloc_unit 1048576 hint 0 -228> 2017-09-22 09:46:37.954221 7f6903e8dd00 4 rocksdb: [/home/zstor/ceph/rpmbuild/BUILD/ceph-12.2.0/src/rocksdb/db/db_impl_open.cc:1063] DB pointer 0x7f69061bc000 -227> 2017-09-22 09:46:37.954257 7f6903e8dd00 1 bluestore(/var/lib/ceph/osd/ceph-19/) _open_db opened rocksdb path db options compression=kNoCompression,max_write_buffer_number=4,min_write_buffer_number_to_merge=1,recycle_log_file_num=4,write_buffer_size=268435456,writable_file_max_buffer_size=0,compaction_readahead_size=2097152 -226> 2017-09-22 09:46:37.954391 7f68eb7c8700 3 rocksdb: [/home/zstor/ceph/rpmbuild/BUILD/ceph-12.2.0/src/rocksdb/db/db_impl.cc:447] ------- DUMPING STATS ------- -225> 2017-09-22 09:46:37.954396 7f68eb7c8700 3 rocksdb: [/home/zstor/ceph/rpmbuild/BUILD/ceph-12.2.0/src/rocksdb/db/db_impl.cc:448] ** DB Stats ** Uptime(secs): 0.0 total, 0.0 interval Cumulative writes: 0 writes, 0 keys, 0 commit groups, 0.0 writes per commit group, ingest: 0.00 GB, 0.00 MB/s Cumulative WAL: 0 writes, 0 syncs, 0.00 writes per sync, written: 0.00 GB, 0.00 MB/s Cumulative stall: 00:00:0.000 H:M:S, 0.0 percent Interval writes: 0 writes, 0 keys, 0 commit groups, 0.0 writes per commit group, ingest: 0.00 MB, 0.00 MB/s Interval WAL: 0 writes, 0 syncs, 0.00 writes per sync, written: 0.00 MB, 0.00 MB/s Interval stall: 00:00:0.000 H:M:S, 0.0 percent ** Compaction Stats [default] ** Level Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop ---------------------------------------------------------------------------------------------------------------------------------------------------------- L0 1/0 36.81 MB 0.2 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0 0 0.000 0 0 L1 4/0 252.59 MB 1.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0 0 0.000 0 0 L2 44/0 2.54 GB 1.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0 0 0.000 0 0 L3 13/0 842.16 MB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0 0 0.000 0 0 Sum 62/0 3.65 GB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0 0 0.000 0 0 Int 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0 0 0.000 0 0 Uptime(secs): 0.0 total, 0.0 interval Flush(GB): cumulative 0.000, interval 0.000 AddFile(GB): cumulative 0.000, interval 0.000 AddFile(Total Files): cumulative 0, interval 0 AddFile(L0 Files): cumulative 0, interval 0 AddFile(Keys): cumulative 0, interval 0 Cumulative compaction: 0.00 GB write, 0.00 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds Interval compaction: 0.00 GB write, 0.00 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds Stalls(count): 0 level0_slowdown, 0 level0_slowdown_with_compaction, 0 level0_numfiles, 0 level0_numfiles_with_compaction, 0 stop for pending_compaction_bytes, 0 slowdown for pending_compaction_bytes, 0 memtable_compaction, 0 memtable_slowdown, interval 0 total count ** File Read Latency Histogram By Level [default] ** ** Compaction Stats [default] ** Level Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop ---------------------------------------------------------------------------------------------------------------------------------------------------------- L0 1/0 36.81 MB 0.2 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0 0 0.000 0 0 L1 4/0 252.59 MB 1.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0 0 0.000 0 0 L2 44/0 2.54 GB 1.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0 0 0.000 0 0 L3 13/0 842.16 MB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0 0 0.000 0 0 Sum 62/0 3.65 GB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0 0 0.000 0 0 Int 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0 0 0.000 0 0 Uptime(secs): 0.0 total, 0.0 interval Flush(GB): cumulative 0.000, interval 0.000 AddFile(GB): cumulative 0.000, interval 0.000 AddFile(Total Files): cumulative 0, interval 0 AddFile(L0 Files): cumulative 0, interval 0 AddFile(Keys): cumulative 0, interval 0 Cumulative compaction: 0.00 GB write, 0.00 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds Interval compaction: 0.00 GB write, 0.00 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds Stalls(count): 0 level0_slowdown, 0 level0_slowdown_with_compaction, 0 level0_numfiles, 0 level0_numfiles_with_compaction, 0 stop for pending_compaction_bytes, 0 slowdown for pending_compaction_bytes, 0 memtable_compaction, 0 memtable_slowdown, interval 0 total count ** File Read Latency Histogram By Level [default] ** -224> 2017-09-22 09:46:37.954426 7f68eb7c8700 4 rocksdb: [/home/zstor/ceph/rpmbuild/BUILD/ceph-12.2.0/src/rocksdb/db/compaction_picker.cc:461] [default] Expanding@2 1+2(69564368+145598984 bytes) to 2+2 (81603618+145598984bytes) -223> 2017-09-22 09:46:37.954462 7f68eb7c8700 4 rocksdb: [/home/zstor/ceph/rpmbuild/BUILD/ceph-12.2.0/src/rocksdb/db/compaction_job.cc:1403] [default] [JOB 3] Compacting 2@2 + 2@3 files to L3, score 1.02 -222> 2017-09-22 09:46:37.954476 7f68eb7c8700 4 rocksdb: [/home/zstor/ceph/rpmbuild/BUILD/ceph-12.2.0/src/rocksdb/db/compaction_job.cc:1407] [default] Compaction start summary: Base version 2 Base level 2, inputs: [1528(66MB) 1529(11MB)], [628(69MB) 680(69MB)] -221> 2017-09-22 09:46:37.954493 7f68eb7c8700 4 rocksdb: EVENT_LOG_v1 {"time_micros": 1506044797954478, "job": 3, "event": "compaction_started", "files_L2": [1528, 1529], "files_L3": [628, 680], "score": 1.01793, "input_data_size": 227202602} -220> 2017-09-22 09:46:37.954581 7f6903e8dd00 10 bluestore(/var/lib/ceph/osd/ceph-19/) _open_super_meta old nid_max 712666 -219> 2017-09-22 09:46:37.954597 7f6903e8dd00 10 bluestore(/var/lib/ceph/osd/ceph-19/) _open_super_meta old blobid_max 163840 -218> 2017-09-22 09:46:38.063809 7f6903e8dd00 10 bluestore(/var/lib/ceph/osd/ceph-19/) _open_super_meta freelist_type bitmap -217> 2017-09-22 09:46:38.063862 7f6903e8dd00 10 bluestore(/var/lib/ceph/osd/ceph-19/) _open_super_meta bluefs_extents 0x[1bf1c600000~2542600000] -216> 2017-09-22 09:46:38.063884 7f6903e8dd00 10 bluestore(/var/lib/ceph/osd/ceph-19/) _open_super_meta ondisk_format 2 compat_ondisk_format 2 -215> 2017-09-22 09:46:38.063899 7f6903e8dd00 10 bluestore(/var/lib/ceph/osd/ceph-19/) _open_super_meta min_alloc_size 0x400000 -214> 2017-09-22 09:46:38.064313 7f6903e8dd00 10 bluestore(/var/lib/ceph/osd/ceph-19/) _set_alloc_sizes min_alloc_size 0x400000 order  max_alloc_size 0x0 prefer_deferred_size 0x0 deferred_batch_ops 16 -213> 2017-09-22 09:46:38.064322 7f6903e8dd00 10 bluestore(/var/lib/ceph/osd/ceph-19/) _set_throttle_params throttle_cost_per_io 4000 -212> 2017-09-22 09:46:38.064325 7f6903e8dd00 10 bluestore(/var/lib/ceph/osd/ceph-19/) _set_csum csum_type crc32c -211> 2017-09-22 09:46:38.064338 7f6903e8dd00 10 bluestore(/var/lib/ceph/osd/ceph-19/) _set_compression compression mode set to 'none', ignore other compression setttings -210> 2017-09-22 09:46:38.064340 7f6903e8dd00 10 bluestore(/var/lib/ceph/osd/ceph-19/) _set_blob_size max_blob_size 0x400000 -209> 2017-09-22 09:46:38.064349 7f6903e8dd00 1 freelist init -208> 2017-09-22 09:46:38.103853 7f6903e8dd00 10 freelist init blocks 0x3a37b300 -207> 2017-09-22 09:46:38.103882 7f6903e8dd00 10 freelist init blocks_per_key 0x80 -206> 2017-09-22 09:46:38.103886 7f6903e8dd00 10 freelist init bytes_per_block 0x1000 -205> 2017-09-22 09:46:38.103890 7f6903e8dd00 10 freelist init size 0x3a37b2cf000 -204> 2017-09-22 09:46:38.103892 7f6903e8dd00 10 freelist init size 0x3a37b2cf000 bytes_per_block 0x1000 blocks 0x3a37b300 blocks_per_key 0x80 -203> 2017-09-22 09:46:38.103906 7f6903e8dd00 10 freelist _init_misc bytes_per_key 0x80000, key_mask 0xfffffffffff80000 -202> 2017-09-22 09:46:38.104606 7f6903e8dd00 10 bitmapalloc:BitMapAllocator instance 140089048693184 size 0x3a37b2cf000 -201> 2017-09-22 09:46:38.104613 7f6903e8dd00 1 bluestore(/var/lib/ceph/osd/ceph-19/) _open_alloc opening allocation metadata -200> 2017-09-22 09:46:38.104619 7f6903e8dd00 10 freelist enumerate_next start -199> 2017-09-22 09:46:38.104683 7f6903e8dd00 10 freelist enumerate_next 0x2000~3fe000 -198> 2017-09-22 09:46:38.104688 7f6903e8dd00 10 bitmapalloc:init_add_free instance 140089048693184 offset 0x2000 length 0x3fe000 -197> 2017-09-22 09:46:38.104691 7f6903e8dd00 20 bitmapalloc:insert_free instance 140089048693184 off 0x400000 len 0x0 -196> 2017-09-22 09:46:38.683243 7f68eb7c8700 10 bitmapalloc:reserve instance 140089048683008 num_used 3663583 total 3815346 -195> 2017-09-22 09:46:38.683254 7f68eb7c8700 10 bitmapalloc:allocate instance 140089048683008 want_size 68157440 alloc_unit 1048576 hint 0 -194> 2017-09-22 09:46:39.117189 7f68eb7c8700 10 bitmapalloc:release 0xd00000~100000 -193> 2017-09-22 09:46:39.117201 7f68eb7c8700 20 bitmapalloc:insert_free instance 140089048682960 off 0xd00000 len 0x100000 -192> 2017-09-22 09:46:39.117271 7f68eb7c8700 3 rocksdb: [/home/zstor/ceph/rpmbuild/BUILD/ceph-12.2.0/src/rocksdb/db/db_impl_compaction_flush.cc:1591] Compaction error: Corruption: block checksum mismatch -191> 2017-09-22 09:46:39.117284 7f68eb7c8700 4 rocksdb: (Original Log Time 2017/09/22-09:46:39.117233) [/home/zstor/ceph/rpmbuild/BUILD/ceph-12.2.0/src/rocksdb/db/compaction_job.cc:621] [default] compacted to: base level 1 max bytes base 268435456 files[1 4 44 13 0 0 0] max score 0.99, MB/sec: 195.4 rd, 57.7 wr, level 3, files in(2, 2) out(1) MB in(77.8, 138.9) out(64.0), read-write-amplify(3.6) write-amplify(0.8) Corruption: block checksum mismatch, records in: 5211814, records dropped: 528 -190> 2017-09-22 09:46:39.117290 7f68eb7c8700 4 rocksdb: (Original Log Time 2017/09/22-09:46:39.117264) EVENT_LOG_v1 {"time_micros": 1506044799117253, "job": 3, "event": "compaction_finished", "compaction_time_micros": 1162712, "output_level": 3, "num_output_files": 1, "total_output_size": 67110452, "num_input_records": 558271, "num_output_records": 557743, "num_subcompactions": 1, "num_single_delete_mismatches": 0, "num_single_delete_fallthrough": 0, "lsm_state": [1, 4, 44, 13, 0, 0, 0]} -189> 2017-09-22 09:46:39.117293 7f68eb7c8700 2 rocksdb: [/home/zstor/ceph/rpmbuild/BUILD/ceph-12.2.0/src/rocksdb/db/db_impl_compaction_flush.cc:1275] Waiting after background compaction error: Corruption: block checksum mismatch, Accumulated background error counts: 1 -188> 2017-09-22 09:46:40.117482 7f68eb7c8700 5 rocksdb: [/home/zstor/ceph/rpmbuild/BUILD/ceph-12.2.0/src/rocksdb/db/db_impl_files.cc:307] [JOB 3] Delete db.slow/001685.sst type=2 #1685 -- OK -187> 2017-09-22 09:46:40.117512 7f68eb7c8700 4 rocksdb: EVENT_LOG_v1 {"time_micros": 1506044800117501, "job": 3, "event": "table_file_deletion", "file_number": 1685} -186> 2017-09-22 09:46:45.162235 7f6903e8dd00 10 freelist enumerate_next 0x1bf1c800000~2542400000 -185> 2017-09-22 09:46:45.162261 7f6903e8dd00 10 bitmapalloc:init_add_free instance 140089048693184 offset 0x1bf1c800000 length 0x2542400000 -184> 2017-09-22 09:46:45.162263 7f6903e8dd00 20 bitmapalloc:insert_free instance 140089048693184 off 0x1bf1c800000 len 0x2542400000 -183> 2017-09-22 09:46:50.184371 7f6903e8dd00 10 freelist enumerate_next 0x2cd60c00000~9f400000 -182> 2017-09-22 09:46:50.184386 7f6903e8dd00 10 bitmapalloc:init_add_free instance 140089048693184 offset 0x2cd60c00000 length 0x9f400000 -181> 2017-09-22 09:46:50.184388 7f6903e8dd00 20 bitmapalloc:insert_free instance 140089048693184 off 0x2cd60c00000 len 0x9f400000 -180> 2017-09-22 09:46:50.184443 7f6903e8dd00 10 freelist enumerate_next 0x2ce00800000~d57aacf000 -179> 2017-09-22 09:46:50.184444 7f6903e8dd00 10 bitmapalloc:init_add_free instance 140089048693184 offset 0x2ce00800000 length 0xd57aacf000 -178> 2017-09-22 09:46:50.184453 7f6903e8dd00 20 bitmapalloc:insert_free instance 140089048693184 off 0x2ce00800000 len 0xd57a800000 -177> 2017-09-22 09:46:50.184568 7f6903e8dd00 10 freelist enumerate_next end -176> 2017-09-22 09:46:50.184578 7f6903e8dd00 1 bluestore(/var/lib/ceph/osd/ceph-19/) _open_alloc loaded 1005 G in 4 extents -175> 2017-09-22 09:46:50.184591 7f6903e8dd00 10 bitmapalloc:init_rm_free instance 140089048693184 offset 0x1bf1c600000 length 0x2542600000 -174> 2017-09-22 09:46:50.184614 7f6903e8dd00 10 bluestore(/var/lib/ceph/osd/ceph-19/) _open_alloc marked bluefs_extents 0x[1bf1c600000~2542600000] as allocated -173> 2017-09-22 09:46:50.184965 7f6903e8dd00 20 bluestore(/var/lib/ceph/osd/ceph-19/) _open_collections opened 3.0_head 0x7f6905d1a8c0 -172> 2017-09-22 09:46:50.184999 7f6903e8dd00 20 bluestore(/var/lib/ceph/osd/ceph-19/) _open_collections opened 3.100_head 0x7f690611aa00 -171> 2017-09-22 09:46:50.185025 7f6903e8dd00 20 bluestore(/var/lib/ceph/osd/ceph-19/) _open_collections opened 3.101_head 0x7f690611b800 -170> 2017-09-22 09:46:50.185031 7f6903e8dd00 20 bluestore(/var/lib/ceph/osd/ceph-19/) _open_collections opened 3.103_head 0x7f690611b480 -169> 2017-09-22 09:46:50.185036 7f6903e8dd00 20 bluestore(/var/lib/ceph/osd/ceph-19/) _open_collections opened 3.104_head 0x7f690611a140 -168> 2017-09-22 09:46:50.185039 7f6903e8dd00 20 bluestore(/var/lib/ceph/osd/ceph-19/) _open_collections opened 3.105_head 0x7f6906119c00 -167> 2017-09-22 09:46:50.185048 7f6903e8dd00 20 bluestore(/var/lib/ceph/osd/ceph-19/) _open_collections opened 3.108_head 0x7f69061a5a40 -166> 2017-09-22 09:46:50.185053 7f6903e8dd00 20 bluestore(/var/lib/ceph/osd/ceph-19/) _open_collections opened 3.10_head 0x7f690611bd40 -165> 2017-09-22 09:46:50.185057 7f6903e8dd00 20 bluestore(/var/lib/ceph/osd/ceph-19/) _open_collections opened 3.10b_head 0x7f690611b100 -164> 2017-09-22 09:46:50.185061 7f6903e8dd00 20 bluestore(/var/lib/ceph/osd/ceph-19/) _open_collections opened 3.10f_head 0x7f690611af40 -163> 2017-09-22 09:46:50.185064 7f6903e8dd00 20 bluestore(/var/lib/ceph/osd/ceph-19/) _open_collections opened 3.110_head 0x7f690611a680 -162> 2017-09-22 09:46:50.185075 7f6903e8dd00 20 bluestore(/var/lib/ceph/osd/ceph-19/) _open_collections opened 3.114_head 0x7f690611a4c0 -161> 2017-09-22 09:46:50.185087 7f6903e8dd00 20 bluestore(/var/lib/ceph/osd/ceph-19/) _open_collections opened 3.115_head 0x7f69061a5880 -160> 2017-09-22 09:46:50.185091 7f6903e8dd00 20 bluestore(/var/lib/ceph/osd/ceph-19/) _open_collections opened 3.116_head 0x7f69061a5340 -159> 2017-09-22 09:46:50.185094 7f6903e8dd00 20 bluestore(/var/lib/ceph/osd/ceph-19/) _open_collections opened 3.118_head 0x7f69061a5180 -158> 2017-09-22 09:46:50.185096 7f6903e8dd00 20 bluestore(/var/lib/ceph/osd/ceph-19/) _open_collections opened 3.11a_head 0x7f69061a4e00 -157> 2017-09-22 09:46:50.185099 7f6903e8dd00 20 bluestore(/var/lib/ceph/osd/ceph-19/) _open_collections opened 3.11c_head 0x7f69061a4c40 -156> 2017-09-22 09:46:50.185102 7f6903e8dd00 20 bluestore(/var/lib/ceph/osd/ceph-19/) _open_collections opened 3.11f_head 0x7f69061a48c0 -155> 2017-09-22 09:46:50.185105 7f6903e8dd00 20 bluestore(/var/lib/ceph/osd/ceph-19/) _open_collections opened 3.12_head 0x7f69061a4700 -154> 2017-09-22 09:46:50.185113 7f6903e8dd00 20 bluestore(/var/lib/ceph/osd/ceph-19/) _open_collections opened 3.16_head 0x7f69061a4380 -153> 2017-09-22 09:46:50.185116 7f6903e8dd00 20 bluestore(/var/lib/ceph/osd/ceph-19/) _open_collections opened 3.18_head 0x7f69061a41c0 -152> 2017-09-22 09:46:50.185119 7f6903e8dd00 20 bluestore(/var/lib/ceph/osd/ceph-19/) _open_collections opened 3.1_head 0x7f69061a5dc0 -151> 2017-09-22 09:46:50.185122 7f6903e8dd00 20 bluestore(/var/lib/ceph/osd/ceph-19/) _open_collections opened 3.1b_head 0x7f69061a5f80 -150> 2017-09-22 09:46:50.185127 7f6903e8dd00 20 bluestore(/var/lib/ceph/osd/ceph-19/) _open_collections opened 3.1f_head 0x7f69061a6140 -149> 2017-09-22 09:46:50.185145 7f6903e8dd00 20 bluestore(/var/lib/ceph/osd/ceph-19/) _open_collections opened 3.22_head 0x7f69061a6300 -148> 2017-09-22 09:46:50.185149 7f6903e8dd00 20 bluestore(/var/lib/ceph/osd/ceph-19/) _open_collections opened 3.25_head 0x7f69061a64c0 -147> 2017-09-22 09:46:50.185152 7f6903e8dd00 20 bluestore(/var/lib/ceph/osd/ceph-19/) _open_collections opened 3.2_head 0x7f69061a6680 -146> 2017-09-22 09:46:50.185160 7f6903e8dd00 20 bluestore(/var/lib/ceph/osd/ceph-19/) _open_collections opened 3.2a_head 0x7f69061a6840 -145> 2017-09-22 09:46:50.185163 7f6903e8dd00 20 bluestore(/var/lib/ceph/osd/ceph-19/) _open_collections opened 3.2d_head 0x7f69061a6a00 -144> 2017-09-22 09:46:50.185166 7f6903e8dd00 20 bluestore(/var/lib/ceph/osd/ceph-19/) _open_collections opened 3.2e_head 0x7f69061a6bc0 -143> 2017-09-22 09:46:50.185169 7f6903e8dd00 20 bluestore(/var/lib/ceph/osd/ceph-19/) _open_collections opened 3.2f_head 0x7f69061a6d80 -142> 2017-09-22 09:46:50.185172 7f6903e8dd00 20 bluestore(/var/lib/ceph/osd/ceph-19/) _open_collections opened 3.30_head 0x7f69061a6f40 -141> 2017-09-22 09:46:50.185174 7f6903e8dd00 20 bluestore(/var/lib/ceph/osd/ceph-19/) _open_collections opened 3.31_head 0x7f69061a7100 -140> 2017-09-22 09:46:50.185177 7f6903e8dd00 20 bluestore(/var/lib/ceph/osd/ceph-19/) _open_collections opened 3.34_head 0x7f69061a72c0 -139> 2017-09-22 09:46:50.185187 7f6903e8dd00 20 bluestore(/var/lib/ceph/osd/ceph-19/) _open_collections opened 3.37_head 0x7f69061a7480 -138> 2017-09-22 09:46:50.185196 7f6903e8dd00 20 bluestore(/var/lib/ceph/osd/ceph-19/) _open_collections opened 3.38_head 0x7f69061a7640 -137> 2017-09-22 09:46:50.185199 7f6903e8dd00 20 bluestore(/var/lib/ceph/osd/ceph-19/) _open_collections opened 3.3a_head 0x7f69061a7800 -136> 2017-09-22 09:46:50.185202 7f6903e8dd00 20 bluestore(/var/lib/ceph/osd/ceph-19/) _open_collections opened 3.3b_head 0x7f69061a79c0 -135> 2017-09-22 09:46:50.185205 7f6903e8dd00 20 bluestore(/var/lib/ceph/osd/ceph-19/) _open_collections opened 3.3d_head 0x7f69061a7b80 -134> 2017-09-22 09:46:50.185208 7f6903e8dd00 20 bluestore(/var/lib/ceph/osd/ceph-19/) _open_collections opened 3.41_head 0x7f69061a7d40 -133> 2017-09-22 09:46:50.185218 7f6903e8dd00 20 bluestore(/var/lib/ceph/osd/ceph-19/) _open_collections opened 3.45_head 0x7f6911f82000 -132> 2017-09-22 09:46:50.185221 7f6903e8dd00 20 bluestore(/var/lib/ceph/osd/ceph-19/) _open_collections opened 3.46_head 0x7f6911f821c0 -131> 2017-09-22 09:46:50.185224 7f6903e8dd00 20 bluestore(/var/lib/ceph/osd/ceph-19/) _open_collections opened 3.4c_head 0x7f6911f82380 -130> 2017-09-22 09:46:50.185227 7f6903e8dd00 20 bluestore(/var/lib/ceph/osd/ceph-19/) _open_collections opened 3.4f_head 0x7f6911f82540 -129> 2017-09-22 09:46:50.185344 7f6903e8dd00 20 bluestore(/var/lib/ceph/osd/ceph-19/) _open_collections opened 3.51_head 0x7f6911f82700 -128> 2017-09-22 09:46:50.185348 7f6903e8dd00 20 bluestore(/var/lib/ceph/osd/ceph-19/) _open_collections opened 3.53_head 0x7f6911f828c0 -127> 2017-09-22 09:46:50.185351 7f6903e8dd00 20 bluestore(/var/lib/ceph/osd/ceph-19/) _open_collections opened 3.56_head 0x7f6911f82a80 -126> 2017-09-22 09:46:50.185355 7f6903e8dd00 20 bluestore(/var/lib/ceph/osd/ceph-19/) _open_collections opened 3.57_head 0x7f6911f82c40 -125> 2017-09-22 09:46:50.185357 7f6903e8dd00 20 bluestore(/var/lib/ceph/osd/ceph-19/) _open_collections opened 3.59_head 0x7f6911f82e00 -124> 2017-09-22 09:46:50.185360 7f6903e8dd00 20 bluestore(/var/lib/ceph/osd/ceph-19/) _open_collections opened 3.5b_head 0x7f6911f82fc0 -123> 2017-09-22 09:46:50.185363 7f6903e8dd00 20 bluestore(/var/lib/ceph/osd/ceph-19/) _open_collections opened 3.5f_head 0x7f6911f83180 -122> 2017-09-22 09:46:50.185366 7f6903e8dd00 20 bluestore(/var/lib/ceph/osd/ceph-19/) _open_collections opened 3.60_head 0x7f6911f83340 -121> 2017-09-22 09:46:50.185369 7f6903e8dd00 20 bluestore(/var/lib/ceph/osd/ceph-19/) _open_collections opened 3.62_head 0x7f6911f83500 -120> 2017-09-22 09:46:50.185372 7f6903e8dd00 20 bluestore(/var/lib/ceph/osd/ceph-19/) _open_collections opened 3.68_head 0x7f6911f836c0 -119> 2017-09-22 09:46:50.185374 7f6903e8dd00 20 bluestore(/var/lib/ceph/osd/ceph-19/) _open_collections opened 3.69_head 0x7f6911f83880 -118> 2017-09-22 09:46:50.185377 7f6903e8dd00 20 bluestore(/var/lib/ceph/osd/ceph-19/) _open_collections opened 3.6a_head 0x7f6911f83a40 -117> 2017-09-22 09:46:50.185380 7f6903e8dd00 20 bluestore(/var/lib/ceph/osd/ceph-19/) _open_collections opened 3.6d_head 0x7f6911f83c00 -116> 2017-09-22 09:46:50.185383 7f6903e8dd00 20 bluestore(/var/lib/ceph/osd/ceph-19/) _open_collections opened 3.6e_head 0x7f6911f83dc0 -115> 2017-09-22 09:46:50.185385 7f6903e8dd00 20 bluestore(/var/lib/ceph/osd/ceph-19/) _open_collections opened 3.71_head 0x7f6911f83f80 -114> 2017-09-22 09:46:50.185388 7f6903e8dd00 20 bluestore(/var/lib/ceph/osd/ceph-19/) _open_collections opened 3.74_head 0x7f6911f84140 -113> 2017-09-22 09:46:50.185391 7f6903e8dd00 20 bluestore(/var/lib/ceph/osd/ceph-19/) _open_collections opened 3.75_head 0x7f6911f84300 -112> 2017-09-22 09:46:50.185394 7f6903e8dd00 20 bluestore(/var/lib/ceph/osd/ceph-19/) _open_collections opened 3.77_head 0x7f6911f844c0 -111> 2017-09-22 09:46:50.185396 7f6903e8dd00 20 bluestore(/var/lib/ceph/osd/ceph-19/) _open_collections opened 3.80_head 0x7f6911f84680 -110> 2017-09-22 09:46:50.185399 7f6903e8dd00 20 bluestore(/var/lib/ceph/osd/ceph-19/) _open_collections opened 3.84_head 0x7f6911f84840 -109> 2017-09-22 09:46:50.185402 7f6903e8dd00 20 bluestore(/var/lib/ceph/osd/ceph-19/) _open_collections opened 3.86_head 0x7f6911f84a00 -108> 2017-09-22 09:46:50.185405 7f6903e8dd00 20 bluestore(/var/lib/ceph/osd/ceph-19/) _open_collections opened 3.87_head 0x7f6911f84bc0 -107> 2017-09-22 09:46:50.185408 7f6903e8dd00 20 bluestore(/var/lib/ceph/osd/ceph-19/) _open_collections opened 3.89_head 0x7f6911f84d80 -106> 2017-09-22 09:46:50.185410 7f6903e8dd00 20 bluestore(/var/lib/ceph/osd/ceph-19/) _open_collections opened 3.8f_head 0x7f6911f84f40 -105> 2017-09-22 09:46:50.185413 7f6903e8dd00 20 bluestore(/var/lib/ceph/osd/ceph-19/) _open_collections opened 3.90_head 0x7f6911f85100 -104> 2017-09-22 09:46:50.185415 7f6903e8dd00 20 bluestore(/var/lib/ceph/osd/ceph-19/) _open_collections opened 3.91_head 0x7f6911f852c0 -103> 2017-09-22 09:46:50.185418 7f6903e8dd00 20 bluestore(/var/lib/ceph/osd/ceph-19/) _open_collections opened 3.95_head 0x7f6911f85480 -102> 2017-09-22 09:46:50.185428 7f6903e8dd00 20 bluestore(/var/lib/ceph/osd/ceph-19/) _open_collections opened 3.96_head 0x7f6911f85640 -101> 2017-09-22 09:46:50.185431 7f6903e8dd00 20 bluestore(/var/lib/ceph/osd/ceph-19/) _open_collections opened 3.99_head 0x7f6911f85800 -100> 2017-09-22 09:46:50.185434 7f6903e8dd00 20 bluestore(/var/lib/ceph/osd/ceph-19/) _open_collections opened 3.9_head 0x7f6911f859c0 -99> 2017-09-22 09:46:50.185437 7f6903e8dd00 20 bluestore(/var/lib/ceph/osd/ceph-19/) _open_collections opened 3.a1_head 0x7f6911f85b80 -98> 2017-09-22 09:46:50.185439 7f6903e8dd00 20 bluestore(/var/lib/ceph/osd/ceph-19/) _open_collections opened 3.a3_head 0x7f6911f85d40 -97> 2017-09-22 09:46:50.185452 7f6903e8dd00 20 bluestore(/var/lib/ceph/osd/ceph-19/) _open_collections opened 3.a6_head 0x7f6908d06000 -96> 2017-09-22 09:46:50.185455 7f6903e8dd00 20 bluestore(/var/lib/ceph/osd/ceph-19/) _open_collections opened 3.a7_head 0x7f6908d061c0 -95> 2017-09-22 09:46:50.185458 7f6903e8dd00 20 bluestore(/var/lib/ceph/osd/ceph-19/) _open_collections opened 3.a_head 0x7f6908d06380 -94> 2017-09-22 09:46:50.185460 7f6903e8dd00 20 bluestore(/var/lib/ceph/osd/ceph-19/) _open_collections opened 3.ac_head 0x7f6908d06540 -93> 2017-09-22 09:46:50.185463 7f6903e8dd00 20 bluestore(/var/lib/ceph/osd/ceph-19/) _open_collections opened 3.ad_head 0x7f6908d06700 -92> 2017-09-22 09:46:50.185466 7f6903e8dd00 20 bluestore(/var/lib/ceph/osd/ceph-19/) _open_collections opened 3.b2_head 0x7f6908d068c0 -91> 2017-09-22 09:46:50.185468 7f6903e8dd00 20 bluestore(/var/lib/ceph/osd/ceph-19/) _open_collections opened 3.b3_head 0x7f6908d06a80 -90> 2017-09-22 09:46:50.185471 7f6903e8dd00 20 bluestore(/var/lib/ceph/osd/ceph-19/) _open_collections opened 3.b8_head 0x7f6908d06c40 -89> 2017-09-22 09:46:50.185474 7f6903e8dd00 20 bluestore(/var/lib/ceph/osd/ceph-19/) _open_collections opened 3.b9_head 0x7f6908d06e00 -88> 2017-09-22 09:46:50.185477 7f6903e8dd00 20 bluestore(/var/lib/ceph/osd/ceph-19/) _open_collections opened 3.be_head 0x7f6908d06fc0 -87> 2017-09-22 09:46:50.185479 7f6903e8dd00 20 bluestore(/var/lib/ceph/osd/ceph-19/) _open_collections opened 3.bf_head 0x7f6908d07180 -86> 2017-09-22 09:46:50.185482 7f6903e8dd00 20 bluestore(/var/lib/ceph/osd/ceph-19/) _open_collections opened 3.c3_head 0x7f6908d07340 -85> 2017-09-22 09:46:50.185485 7f6903e8dd00 20 bluestore(/var/lib/ceph/osd/ceph-19/) _open_collections opened 3.c6_head 0x7f6908d07500 -84> 2017-09-22 09:46:50.185492 7f6903e8dd00 20 bluestore(/var/lib/ceph/osd/ceph-19/) _open_collections opened 3.cb_head 0x7f6908d076c0 -83> 2017-09-22 09:46:50.185495 7f6903e8dd00 20 bluestore(/var/lib/ceph/osd/ceph-19/) _open_collections opened 3.d0_head 0x7f6908d07880 -82> 2017-09-22 09:46:50.185498 7f6903e8dd00 20 bluestore(/var/lib/ceph/osd/ceph-19/) _open_collections opened 3.d1_head 0x7f6908d07a40 -81> 2017-09-22 09:46:50.185507 7f6903e8dd00 20 bluestore(/var/lib/ceph/osd/ceph-19/) _open_collections opened 3.e0_head 0x7f6908d07c00 -80> 2017-09-22 09:46:50.185510 7f6903e8dd00 20 bluestore(/var/lib/ceph/osd/ceph-19/) _open_collections opened 3.e3_head 0x7f6908d07dc0 -79> 2017-09-22 09:46:50.185513 7f6903e8dd00 20 bluestore(/var/lib/ceph/osd/ceph-19/) _open_collections opened 3.e6_head 0x7f6908d07f80 -78> 2017-09-22 09:46:50.185515 7f6903e8dd00 20 bluestore(/var/lib/ceph/osd/ceph-19/) _open_collections opened 3.e8_head 0x7f6908d08140 -77> 2017-09-22 09:46:50.185518 7f6903e8dd00 20 bluestore(/var/lib/ceph/osd/ceph-19/) _open_collections opened 3.ec_head 0x7f6908d08300 -76> 2017-09-22 09:46:50.185529 7f6903e8dd00 20 bluestore(/var/lib/ceph/osd/ceph-19/) _open_collections opened 3.ef_head 0x7f6908d084c0 -75> 2017-09-22 09:46:50.185532 7f6903e8dd00 20 bluestore(/var/lib/ceph/osd/ceph-19/) _open_collections opened 3.f3_head 0x7f6908d08680 -74> 2017-09-22 09:46:50.185534 7f6903e8dd00 20 bluestore(/var/lib/ceph/osd/ceph-19/) _open_collections opened 3.f4_head 0x7f6908d08840 -73> 2017-09-22 09:46:50.185537 7f6903e8dd00 20 bluestore(/var/lib/ceph/osd/ceph-19/) _open_collections opened 3.f6_head 0x7f6908d08a00 -72> 2017-09-22 09:46:50.185539 7f6903e8dd00 20 bluestore(/var/lib/ceph/osd/ceph-19/) _open_collections opened 3.f9_head 0x7f6908d08bc0 -71> 2017-09-22 09:46:50.185542 7f6903e8dd00 20 bluestore(/var/lib/ceph/osd/ceph-19/) _open_collections opened 3.fa_head 0x7f6908d08d80 -70> 2017-09-22 09:46:50.185544 7f6903e8dd00 20 bluestore(/var/lib/ceph/osd/ceph-19/) _open_collections opened 3.fd_head 0x7f6908d08f40 -69> 2017-09-22 09:46:50.185547 7f6903e8dd00 20 bluestore(/var/lib/ceph/osd/ceph-19/) _open_collections opened 3.fe_head 0x7f6908d09100 -68> 2017-09-22 09:46:50.185550 7f6903e8dd00 20 bluestore(/var/lib/ceph/osd/ceph-19/) _open_collections opened meta 0x7f6908d092c0 -67> 2017-09-22 09:46:50.185624 7f6903e8dd00 10 bluestore(/var/lib/ceph/osd/ceph-19/) _kv_start -66> 2017-09-22 09:46:50.185686 7f68ebfc9700 10 trim shard target 2560 M meta/data ratios 0.833333 + 1.49012e-08 (2133 M + 40 ), current 0 (0 + 0 ) -65> 2017-09-22 09:46:50.185939 7f6903e8dd00 10 bluestore(/var/lib/ceph/osd/ceph-19/) _deferred_replay start -64> 2017-09-22 09:46:50.185949 7f68f07d2700 10 bluestore(/var/lib/ceph/osd/ceph-19/) _kv_sync_thread start -63> 2017-09-22 09:46:50.185965 7f68f07d2700 20 bluestore(/var/lib/ceph/osd/ceph-19/) _kv_sync_thread sleep -62> 2017-09-22 09:46:50.185971 7f68effd1700 10 bluestore(/var/lib/ceph/osd/ceph-19/) _kv_finalize_thread start -61> 2017-09-22 09:46:50.185976 7f68effd1700 20 bluestore(/var/lib/ceph/osd/ceph-19/) _kv_finalize_thread sleep -60> 2017-09-22 09:46:50.185972 7f6903e8dd00 20 bluestore(/var/lib/ceph/osd/ceph-19/) _deferred_replay replay 0x000000000000052a -59> 2017-09-22 09:46:50.186033 7f6903e8dd00 20 bluestore(/var/lib/ceph/osd/ceph-19/) _txc_create osr 0x7f6906126060 = 0x7f6905fb0dc0 seq 1 -58> 2017-09-22 09:46:50.186041 7f6903e8dd00 10 bluestore(/var/lib/ceph/osd/ceph-19/) _txc_state_proc txc 0x7f6905fb0dc0 kv_done -57> 2017-09-22 09:46:50.186044 7f6903e8dd00 20 bluestore(/var/lib/ceph/osd/ceph-19/) _deferred_queue txc 0x7f6905fb0dc0 osr 0x7f6906126060 -56> 2017-09-22 09:46:50.186080 7f6903e8dd00 20 bluestore.DeferredBatch(0x7f690616b560) prepare_write seq 1322 0x2cc8fc04000~2000 crc c2a954b4 -55> 2017-09-22 09:46:50.186112 7f6903e8dd00 20 bluestore.DeferredBatch(0x7f690616b560) prepare_write seq 1322 0x2cc8fc10000~1000 crc 558b6240 -54> 2017-09-22 09:46:50.186894 7f6903e8dd00 20 bluestore(/var/lib/ceph/osd/ceph-19/) _deferred_replay replay 0x000000000000052b -53> 2017-09-22 09:46:50.186914 7f6903e8dd00 20 bluestore(/var/lib/ceph/osd/ceph-19/) _txc_create osr 0x7f6906126060 = 0x7f6905fb0b00 seq 2 -52> 2017-09-22 09:46:50.186916 7f6903e8dd00 10 bluestore(/var/lib/ceph/osd/ceph-19/) _txc_state_proc txc 0x7f6905fb0b00 kv_done -51> 2017-09-22 09:46:50.186917 7f6903e8dd00 20 bluestore(/var/lib/ceph/osd/ceph-19/) _deferred_queue txc 0x7f6905fb0b00 osr 0x7f6906126060 -50> 2017-09-22 09:46:50.186921 7f6903e8dd00 20 bluestore.DeferredBatch(0x7f690616b560) prepare_write seq 1323 0x11a3e4a0000~20000 crc c3b40f52 -49> 2017-09-22 09:46:50.187594 7f6903e8dd00 20 bluestore(/var/lib/ceph/osd/ceph-19/) _deferred_replay replay 0x000000000000052c -48> 2017-09-22 09:46:50.187612 7f6903e8dd00 20 bluestore(/var/lib/ceph/osd/ceph-19/) _txc_create osr 0x7f6906126060 = 0x7f6905fb1080 seq 3 -47> 2017-09-22 09:46:50.187614 7f6903e8dd00 10 bluestore(/var/lib/ceph/osd/ceph-19/) _txc_state_proc txc 0x7f6905fb1080 kv_done -46> 2017-09-22 09:46:50.187615 7f6903e8dd00 20 bluestore(/var/lib/ceph/osd/ceph-19/) _deferred_queue txc 0x7f6905fb1080 osr 0x7f6906126060 -45> 2017-09-22 09:46:50.187617 7f6903e8dd00 20 bluestore.DeferredBatch(0x7f690616b560) prepare_write seq 1324 0x11852080000~20000 crc 968a645d -44> 2017-09-22 09:46:50.188266 7f6903e8dd00 20 bluestore(/var/lib/ceph/osd/ceph-19/) _deferred_replay replay 0x000000000000052d -43> 2017-09-22 09:46:50.188285 7f6903e8dd00 20 bluestore(/var/lib/ceph/osd/ceph-19/) _txc_create osr 0x7f6906126060 = 0x7f6905fb0840 seq 4 -42> 2017-09-22 09:46:50.188288 7f6903e8dd00 10 bluestore(/var/lib/ceph/osd/ceph-19/) _txc_state_proc txc 0x7f6905fb0840 kv_done -41> 2017-09-22 09:46:50.188289 7f6903e8dd00 20 bluestore(/var/lib/ceph/osd/ceph-19/) _deferred_queue txc 0x7f6905fb0840 osr 0x7f6906126060 -40> 2017-09-22 09:46:50.188297 7f6903e8dd00 20 bluestore.DeferredBatch(0x7f690616b560) prepare_write seq 1325 0x3cc3400000~20000 crc ae53110f -39> 2017-09-22 09:46:50.188562 7f6903e8dd00 20 bluestore(/var/lib/ceph/osd/ceph-19/) _deferred_replay replay 0x000000000000052e -38> 2017-09-22 09:46:50.188581 7f6903e8dd00 20 bluestore(/var/lib/ceph/osd/ceph-19/) _txc_create osr 0x7f6906126060 = 0x7f6905fb0580 seq 5 -37> 2017-09-22 09:46:50.188582 7f6903e8dd00 10 bluestore(/var/lib/ceph/osd/ceph-19/) _txc_state_proc txc 0x7f6905fb0580 kv_done -36> 2017-09-22 09:46:50.188583 7f6903e8dd00 20 bluestore(/var/lib/ceph/osd/ceph-19/) _deferred_queue txc 0x7f6905fb0580 osr 0x7f6906126060 -35> 2017-09-22 09:46:50.188585 7f6903e8dd00 20 bluestore.DeferredBatch(0x7f690616b560) prepare_write seq 1326 0x2cc91c07000~2000 crc b39fb09b -34> 2017-09-22 09:46:50.188589 7f6903e8dd00 20 bluestore.DeferredBatch(0x7f690616b560) prepare_write seq 1326 0x2cc91c10000~1000 crc cb49bc95 -33> 2017-09-22 09:46:50.197357 7f6903e8dd00 20 bluestore(/var/lib/ceph/osd/ceph-19/) _deferred_replay draining osr -32> 2017-09-22 09:46:50.197365 7f6903e8dd00 10 bluestore(/var/lib/ceph/osd/ceph-19/) _osr_drain_all -31> 2017-09-22 09:46:50.197367 7f6903e8dd00 20 bluestore(/var/lib/ceph/osd/ceph-19/) _osr_drain_all osr_set 0x7f6906126060 -30> 2017-09-22 09:46:50.197369 7f6903e8dd00 20 bluestore(/var/lib/ceph/osd/ceph-19/) deferred_try_submit 1 osrs, 5 txcs -29> 2017-09-22 09:46:50.197371 7f6903e8dd00 10 bluestore(/var/lib/ceph/osd/ceph-19/) _deferred_submit_unlock osr 0x7f6906126060 7 ios pending -28> 2017-09-22 09:46:50.197375 7f6903e8dd00 20 bluestore(/var/lib/ceph/osd/ceph-19/) _deferred_submit_unlock seq 1325 0x3cc3400000~20000 -27> 2017-09-22 09:46:50.197377 7f6903e8dd00 20 bluestore(/var/lib/ceph/osd/ceph-19/) _deferred_submit_unlock write 0x3cc3400000~20000 crc ae53110f -26> 2017-09-22 09:46:50.197459 7f6903e8dd00 20 bluestore(/var/lib/ceph/osd/ceph-19/) _deferred_submit_unlock seq 1324 0x11852080000~20000 -25> 2017-09-22 09:46:50.197462 7f6903e8dd00 20 bluestore(/var/lib/ceph/osd/ceph-19/) _deferred_submit_unlock write 0x11852080000~20000 crc 968a645d -24> 2017-09-22 09:46:50.197533 7f6903e8dd00 20 bluestore(/var/lib/ceph/osd/ceph-19/) _deferred_submit_unlock seq 1323 0x11a3e4a0000~20000 -23> 2017-09-22 09:46:50.197535 7f6903e8dd00 20 bluestore(/var/lib/ceph/osd/ceph-19/) _deferred_submit_unlock write 0x11a3e4a0000~20000 crc c3b40f52 -22> 2017-09-22 09:46:50.197604 7f6903e8dd00 20 bluestore(/var/lib/ceph/osd/ceph-19/) _deferred_submit_unlock seq 1322 0x2cc8fc04000~2000 -21> 2017-09-22 09:46:50.197607 7f6903e8dd00 20 bluestore(/var/lib/ceph/osd/ceph-19/) _deferred_submit_unlock write 0x2cc8fc04000~2000 crc c2a954b4 -20> 2017-09-22 09:46:50.197610 7f6903e8dd00 20 bluestore(/var/lib/ceph/osd/ceph-19/) _deferred_submit_unlock seq 1322 0x2cc8fc10000~1000 -19> 2017-09-22 09:46:50.197612 7f6903e8dd00 20 bluestore(/var/lib/ceph/osd/ceph-19/) _deferred_submit_unlock write 0x2cc8fc10000~1000 crc 558b6240 -18> 2017-09-22 09:46:50.197614 7f6903e8dd00 20 bluestore(/var/lib/ceph/osd/ceph-19/) _deferred_submit_unlock seq 1326 0x2cc91c07000~2000 -17> 2017-09-22 09:46:50.197615 7f6903e8dd00 20 bluestore(/var/lib/ceph/osd/ceph-19/) _deferred_submit_unlock write 0x2cc91c07000~2000 crc b39fb09b -16> 2017-09-22 09:46:50.197621 7f6903e8dd00 20 bluestore(/var/lib/ceph/osd/ceph-19/) _deferred_submit_unlock seq 1326 0x2cc91c10000~1000 -15> 2017-09-22 09:46:50.197623 7f6903e8dd00 20 bluestore(/var/lib/ceph/osd/ceph-19/) _deferred_submit_unlock write 0x2cc91c10000~1000 crc cb49bc95 -14> 2017-09-22 09:46:50.197703 7f6903e8dd00 20 bluestore(/var/lib/ceph/osd/ceph-19/) _osr_drain_all drain 0x7f6906126060 -13> 2017-09-22 09:46:50.197710 7f68f07d2700 20 bluestore(/var/lib/ceph/osd/ceph-19/) _kv_sync_thread wake -12> 2017-09-22 09:46:50.197710 7f68effd1700 20 bluestore(/var/lib/ceph/osd/ceph-19/) _kv_finalize_thread wake -11> 2017-09-22 09:46:50.197722 7f68f07d2700 20 bluestore(/var/lib/ceph/osd/ceph-19/) _kv_sync_thread sleep -10> 2017-09-22 09:46:50.197724 7f68effd1700 20 bluestore(/var/lib/ceph/osd/ceph-19/) _kv_finalize_thread sleep -9> 2017-09-22 09:46:50.199055 7f68f4fdb700 10 bluestore(/var/lib/ceph/osd/ceph-19/) _deferred_aio_finish osr 0x7f6906126060 -8> 2017-09-22 09:46:50.199067 7f68f4fdb700 20 bluestore(/var/lib/ceph/osd/ceph-19/) _deferred_aio_finish dequeueing -7> 2017-09-22 09:46:50.199087 7f68f07d2700 20 bluestore(/var/lib/ceph/osd/ceph-19/) _kv_sync_thread wake -6> 2017-09-22 09:46:50.199095 7f68f07d2700 20 bluestore(/var/lib/ceph/osd/ceph-19/) _kv_sync_thread committing 0 submitting 0 deferred done 1 stable 0 -5> 2017-09-22 09:46:50.199100 7f68f07d2700 20 bluestore(/var/lib/ceph/osd/ceph-19/) _kv_sync_thread num_aios=0 force_flush=1, flushing, deferred done->stable -4> 2017-09-22 09:46:50.316440 7f68f07d2700 10 bluestore(/var/lib/ceph/osd/ceph-19/) _kv_sync_thread new_nid_max 713690 -3> 2017-09-22 09:46:50.316454 7f68f07d2700 10 bluestore(/var/lib/ceph/osd/ceph-19/) _kv_sync_thread new_blobid_max 174080 -2> 2017-09-22 09:46:50.316466 7f68f07d2700 10 bluestore(/var/lib/ceph/osd/ceph-19/) _balance_bluefs_freespace bluefs 148 G free (0.993998) bluestore 856 G free (0.22985), bluefs_ratio 0.147472 -1> 2017-09-22 09:46:50.316534 7f68f07d2700 -1 rocksdb: submit_transaction_sync error: Corruption: block checksum mismatch code = 2 Rocksdb transaction: Put( Prefix = S key = 'nid_max' Value size = 8) Put( Prefix = S key = 'blobid_max' Value size = 8) SingleDelete(Prefix = L Key = 0x000000000000052a) SingleDelete(Prefix = L Key = 0x000000000000052b) SingleDelete(Prefix = L Key = 0x000000000000052c) SingleDelete(Prefix = L Key = 0x000000000000052d) SingleDelete(Prefix = L Key = 0x000000000000052e) 0> 2017-09-22 09:46:50.317302 7f68f07d2700 -1 /home/zstor/ceph/rpmbuild/BUILD/ceph-12.2.0/src/os/bluestore/BlueStore.cc: In function 'void BlueStore::_kv_sync_thread()' thread 7f68f07d2700 time 2017-09-22 09:46:50.316550 /home/zstor/ceph/rpmbuild/BUILD/ceph-12.2.0/src/os/bluestore/BlueStore.cc: 8545: FAILED assert(r == 0) ceph version 12.2.0-3-4-g9a59fd6 (9a59fd686ef3d9445911fae9cdb081c148301a4d) luminous (stable) 1: (ceph::__ceph_assert_fail(char const*, char const*, int, char const*)+0x110) [0x7f68fa57f590] 2: (BlueStore::_kv_sync_thread()+0x35ee) [0x7f69041c147e] 3: (BlueStore::KVSyncThread::entry()+0xd) [0x7f690420982d] 4: (()+0x7df3) [0x7f68f8e42df3] 5: (clone()+0x6d) [0x7f68f79023ed] NOTE: a copy of the executable, or `objdump -rdS ` is needed to interpret this. --- logging levels --- 0/ 5 none 0/ 1 lockdep 0/ 1 context 1/ 1 crush 1/ 5 mds 1/ 5 mds_balancer 1/ 5 mds_locker 1/ 5 mds_log 1/ 5 mds_log_expire 1/ 5 mds_migrator 0/ 1 buffer 0/ 1 timer 0/ 1 filer 0/ 1 striper 0/ 1 objecter 0/ 5 rados 0/ 5 rbd 0/ 5 rbd_mirror 0/ 5 rbd_replay 0/ 5 journaler 0/ 5 objectcacher 0/ 5 client 1/ 5 osd 0/ 5 optracker 0/ 5 objclass 1/ 3 filestore 1/ 3 journal 0/ 5 ms 1/ 5 mon 0/10 monc 1/ 5 paxos 0/ 5 tp 1/ 5 auth 1/ 5 crypto 1/ 1 finisher 1/ 5 heartbeatmap 1/ 5 perfcounter 1/ 5 rgw 1/10 civetweb 1/ 5 javaclient 1/ 5 asok 1/ 1 throttle 0/ 0 refs 1/ 5 xio 1/ 5 compressor 20/20 bluestore 1/ 5 bluefs 1/ 3 bdev 1/ 5 kstore 4/ 5 rocksdb 4/ 5 leveldb 4/ 5 memdb 1/ 5 kinetic 1/ 5 fuse 1/ 5 mgr 1/ 5 mgrc 1/ 5 dpdk 1/ 5 eventtrace -2/-2 (syslog threshold) 99/99 (stderr threshold) max_recent 500 max_new 1000 log_file out --- end dump of recent events --- *** Caught signal (Aborted) ** in thread 7f68f07d2700 thread_name:bstore_kv_sync ceph version 12.2.0-3-4-g9a59fd6 (9a59fd686ef3d9445911fae9cdb081c148301a4d) luminous (stable) 1: (()+0x3dfa51) [0x7f69042a3a51] 2: (()+0xf130) [0x7f68f8e4a130] 3: (gsignal()+0x39) [0x7f68f7841989] 4: (abort()+0x148) [0x7f68f7843098] 5: (ceph::__ceph_assert_fail(char const*, char const*, int, char const*)+0x284) [0x7f68fa57f704] 6: (BlueStore::_kv_sync_thread()+0x35ee) [0x7f69041c147e] 7: (BlueStore::KVSyncThread::entry()+0xd) [0x7f690420982d] 8: (()+0x7df3) [0x7f68f8e42df3] 9: (clone()+0x6d) [0x7f68f79023ed] 2017-09-22 09:46:50.321811 7f68f07d2700 -1 *** Caught signal (Aborted) ** in thread 7f68f07d2700 thread_name:bstore_kv_sync ceph version 12.2.0-3-4-g9a59fd6 (9a59fd686ef3d9445911fae9cdb081c148301a4d) luminous (stable) 1: (()+0x3dfa51) [0x7f69042a3a51] 2: (()+0xf130) [0x7f68f8e4a130] 3: (gsignal()+0x39) [0x7f68f7841989] 4: (abort()+0x148) [0x7f68f7843098] 5: (ceph::__ceph_assert_fail(char const*, char const*, int, char const*)+0x284) [0x7f68fa57f704] 6: (BlueStore::_kv_sync_thread()+0x35ee) [0x7f69041c147e] 7: (BlueStore::KVSyncThread::entry()+0xd) [0x7f690420982d] 8: (()+0x7df3) [0x7f68f8e42df3] 9: (clone()+0x6d) [0x7f68f79023ed] NOTE: a copy of the executable, or `objdump -rdS ` is needed to interpret this. --- begin dump of recent events --- 0> 2017-09-22 09:46:50.321811 7f68f07d2700 -1 *** Caught signal (Aborted) ** in thread 7f68f07d2700 thread_name:bstore_kv_sync ceph version 12.2.0-3-4-g9a59fd6 (9a59fd686ef3d9445911fae9cdb081c148301a4d) luminous (stable) 1: (()+0x3dfa51) [0x7f69042a3a51] 2: (()+0xf130) [0x7f68f8e4a130] 3: (gsignal()+0x39) [0x7f68f7841989] 4: (abort()+0x148) [0x7f68f7843098] 5: (ceph::__ceph_assert_fail(char const*, char const*, int, char const*)+0x284) [0x7f68fa57f704] 6: (BlueStore::_kv_sync_thread()+0x35ee) [0x7f69041c147e] 7: (BlueStore::KVSyncThread::entry()+0xd) [0x7f690420982d] 8: (()+0x7df3) [0x7f68f8e42df3] 9: (clone()+0x6d) [0x7f68f79023ed] NOTE: a copy of the executable, or `objdump -rdS ` is needed to interpret this. --- logging levels --- 0/ 5 none 0/ 1 lockdep 0/ 1 context 1/ 1 crush 1/ 5 mds 1/ 5 mds_balancer 1/ 5 mds_locker 1/ 5 mds_log 1/ 5 mds_log_expire 1/ 5 mds_migrator 0/ 1 buffer 0/ 1 timer 0/ 1 filer 0/ 1 striper 0/ 1 objecter 0/ 5 rados 0/ 5 rbd 0/ 5 rbd_mirror 0/ 5 rbd_replay 0/ 5 journaler 0/ 5 objectcacher 0/ 5 client 1/ 5 osd 0/ 5 optracker 0/ 5 objclass 1/ 3 filestore 1/ 3 journal 0/ 5 ms 1/ 5 mon 0/10 monc 1/ 5 paxos 0/ 5 tp 1/ 5 auth 1/ 5 crypto 1/ 1 finisher 1/ 5 heartbeatmap 1/ 5 perfcounter 1/ 5 rgw 1/10 civetweb 1/ 5 javaclient 1/ 5 asok 1/ 1 throttle 0/ 0 refs 1/ 5 xio 1/ 5 compressor 20/20 bluestore 1/ 5 bluefs 1/ 3 bdev 1/ 5 kstore 4/ 5 rocksdb 4/ 5 leveldb 4/ 5 memdb 1/ 5 kinetic 1/ 5 fuse 1/ 5 mgr 1/ 5 mgrc 1/ 5 dpdk 1/ 5 eventtrace -2/-2 (syslog threshold) 99/99 (stderr threshold) max_recent 500 max_new 1000 log_file out --- end dump of recent events ---