Project

General

Profile

Bug #42223 » osd1-kvstore.txt

Tobias Fischer, 10/28/2019 03:26 PM

 
2019-10-28 16:18:58.573 7f68e5ee6180 1 bluestore(/var/lib/ceph/osd/ceph-1/) _mount path /var/lib/ceph/osd/ceph-1/
2019-10-28 16:18:58.573 7f68e5ee6180 1 bdev create path /var/lib/ceph/osd/ceph-1//block type kernel
2019-10-28 16:18:58.573 7f68e5ee6180 1 bdev(0x556f0325c700 /var/lib/ceph/osd/ceph-1//block) open path /var/lib/ceph/osd/ceph-1//block
2019-10-28 16:18:58.573 7f68e5ee6180 1 bdev(0x556f0325c700 /var/lib/ceph/osd/ceph-1//block) open size 299964039168 (0x45d7400000, 279 GiB) block_size 4096 (4 KiB) rotational discard not supported
2019-10-28 16:18:58.573 7f68e5ee6180 1 bluestore(/var/lib/ceph/osd/ceph-1/) _set_cache_sizes cache_size 1073741824 meta 0.4 kv 0.4 data 0.2
2019-10-28 16:18:58.573 7f68e5ee6180 1 bdev create path /var/lib/ceph/osd/ceph-1//block.db type kernel
2019-10-28 16:18:58.573 7f68e5ee6180 1 bdev(0x556f0325ce00 /var/lib/ceph/osd/ceph-1//block.db) open path /var/lib/ceph/osd/ceph-1//block.db
2019-10-28 16:18:58.573 7f68e5ee6180 1 bdev(0x556f0325ce00 /var/lib/ceph/osd/ceph-1//block.db) open size 28991029248 (0x6c0000000, 27 GiB) block_size 4096 (4 KiB) non-rotational discard not supported
2019-10-28 16:18:58.573 7f68e5ee6180 1 bluefs add_block_device bdev 1 path /var/lib/ceph/osd/ceph-1//block.db size 27 GiB
2019-10-28 16:18:58.573 7f68e5ee6180 1 bdev create path /var/lib/ceph/osd/ceph-1//block type kernel
2019-10-28 16:18:58.573 7f68e5ee6180 1 bdev(0x556f0325d180 /var/lib/ceph/osd/ceph-1//block) open path /var/lib/ceph/osd/ceph-1//block
2019-10-28 16:18:58.573 7f68e5ee6180 1 bdev(0x556f0325d180 /var/lib/ceph/osd/ceph-1//block) open size 299964039168 (0x45d7400000, 279 GiB) block_size 4096 (4 KiB) rotational discard not supported
2019-10-28 16:18:58.573 7f68e5ee6180 1 bluefs add_block_device bdev 2 path /var/lib/ceph/osd/ceph-1//block size 279 GiB
2019-10-28 16:18:58.573 7f68e5ee6180 1 bdev create path /var/lib/ceph/osd/ceph-1//block.wal type kernel
2019-10-28 16:18:58.573 7f68e5ee6180 1 bdev(0x556f0325d500 /var/lib/ceph/osd/ceph-1//block.wal) open path /var/lib/ceph/osd/ceph-1//block.wal
2019-10-28 16:18:58.573 7f68e5ee6180 1 bdev(0x556f0325d500 /var/lib/ceph/osd/ceph-1//block.wal) open size 1073741824 (0x40000000, 1 GiB) block_size 4096 (4 KiB) non-rotational discard not supported
2019-10-28 16:18:58.573 7f68e5ee6180 1 bluefs add_block_device bdev 0 path /var/lib/ceph/osd/ceph-1//block.wal size 1 GiB
2019-10-28 16:18:58.573 7f68e5ee6180 1 bluefs mount
2019-10-28 16:18:58.581 7f68e5ee6180 0 set rocksdb option compaction_readahead_size = 2097152
2019-10-28 16:18:58.581 7f68e5ee6180 0 set rocksdb option compression = kNoCompression
2019-10-28 16:18:58.581 7f68e5ee6180 0 set rocksdb option max_background_compactions = 2
2019-10-28 16:18:58.581 7f68e5ee6180 0 set rocksdb option max_write_buffer_number = 4
2019-10-28 16:18:58.581 7f68e5ee6180 0 set rocksdb option min_write_buffer_number_to_merge = 1
2019-10-28 16:18:58.581 7f68e5ee6180 0 set rocksdb option recycle_log_file_num = 4
2019-10-28 16:18:58.581 7f68e5ee6180 0 set rocksdb option writable_file_max_buffer_size = 0
2019-10-28 16:18:58.581 7f68e5ee6180 0 set rocksdb option write_buffer_size = 268435456
2019-10-28 16:18:58.581 7f68e5ee6180 0 set rocksdb option compaction_readahead_size = 2097152
2019-10-28 16:18:58.581 7f68e5ee6180 0 set rocksdb option compression = kNoCompression
2019-10-28 16:18:58.581 7f68e5ee6180 0 set rocksdb option max_background_compactions = 2
2019-10-28 16:18:58.581 7f68e5ee6180 0 set rocksdb option max_write_buffer_number = 4
2019-10-28 16:18:58.581 7f68e5ee6180 0 set rocksdb option min_write_buffer_number_to_merge = 1
2019-10-28 16:18:58.581 7f68e5ee6180 0 set rocksdb option recycle_log_file_num = 4
2019-10-28 16:18:58.581 7f68e5ee6180 0 set rocksdb option writable_file_max_buffer_size = 0
2019-10-28 16:18:58.581 7f68e5ee6180 0 set rocksdb option write_buffer_size = 268435456
2019-10-28 16:18:58.589 7f68e5ee6180 1 rocksdb: do_open column families: [default]
2019-10-28 16:18:58.777 7f68e5ee6180 1 bluestore(/var/lib/ceph/osd/ceph-1/) _open_db opened rocksdb path db options compression=kNoCompression,max_write_buffer_number=4,min_write_buffer_number_to_merge=1,recycle_log_file_num=4,write_buffer_size=268435456,writable_file_max_buffer_size=0,compaction_readahead_size=2097152,max_background_compactions=2
2019-10-28 16:18:58.785 7f68e5ee6180 1 freelist init
2019-10-28 16:18:58.789 7f68e5ee6180 1 bluestore(/var/lib/ceph/osd/ceph-1/) _open_alloc opening allocation metadata
2019-10-28 16:18:58.789 7f68e5ee6180 1 bluestore(/var/lib/ceph/osd/ceph-1/) _open_alloc loaded 0 B in 0 extents
/data/Debian/WORKDIR/ceph-14.2.4/src/os/bluestore/fastbmap_allocator_impl.h: In function 'void AllocatorLevel02<T>::_mark_allocated(uint64_t, uint64_t) [with L1 = AllocatorLevel01Loose; uint64_t = long unsigned int]' thread 7f68e5ee6180 time 2019-10-28 16:18:58.794917
/data/Debian/WORKDIR/ceph-14.2.4/src/os/bluestore/fastbmap_allocator_impl.h: 750: FAILED ceph_assert(available >= allocated)
ceph version 14.2.4 (75f4de193b3ea58512f204623e6c5a16e6c1e1ba) nautilus (stable)
1: (ceph::__ceph_assert_fail(char const*, char const*, int, char const*)+0x152) [0x7f68e717360e]
2: (()+0x2807e6) [0x7f68e71737e6]
3: (BitmapAllocator::init_rm_free(unsigned long, unsigned long)+0x733) [0x556effc79163]
4: (BlueStore::_open_alloc()+0x1f5) [0x556effa9ff95]
5: (BlueStore::_open_db_and_around(bool)+0xa2) [0x556effab83f2]
6: (BlueStore::_mount(bool, bool)+0x58a) [0x556effafe62a]
7: (StoreTool::load_bluestore(std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > const&, bool)+0x42) [0x556effa6ff22]
8: (StoreTool::StoreTool(std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > const&, std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > const&, bool, bool)+0x226) [0x556effa72f56]
9: (main()+0x28e) [0x556effa4ae4e]
10: (__libc_start_main()+0xeb) [0x7f68e63f309b]
11: (_start()+0x2a) [0x556effa6fdaa]
2019-10-28 16:18:58.793 7f68e5ee6180 -1 /data/Debian/WORKDIR/ceph-14.2.4/src/os/bluestore/fastbmap_allocator_impl.h: In function 'void AllocatorLevel02<T>::_mark_allocated(uint64_t, uint64_t) [with L1 = AllocatorLevel01Loose; uint64_t = long unsigned int]' thread 7f68e5ee6180 time 2019-10-28 16:18:58.794917
/data/Debian/WORKDIR/ceph-14.2.4/src/os/bluestore/fastbmap_allocator_impl.h: 750: FAILED ceph_assert(available >= allocated)

ceph version 14.2.4 (75f4de193b3ea58512f204623e6c5a16e6c1e1ba) nautilus (stable)
1: (ceph::__ceph_assert_fail(char const*, char const*, int, char const*)+0x152) [0x7f68e717360e]
2: (()+0x2807e6) [0x7f68e71737e6]
3: (BitmapAllocator::init_rm_free(unsigned long, unsigned long)+0x733) [0x556effc79163]
4: (BlueStore::_open_alloc()+0x1f5) [0x556effa9ff95]
5: (BlueStore::_open_db_and_around(bool)+0xa2) [0x556effab83f2]
6: (BlueStore::_mount(bool, bool)+0x58a) [0x556effafe62a]
7: (StoreTool::load_bluestore(std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > const&, bool)+0x42) [0x556effa6ff22]
8: (StoreTool::StoreTool(std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > const&, std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > const&, bool, bool)+0x226) [0x556effa72f56]
9: (main()+0x28e) [0x556effa4ae4e]
10: (__libc_start_main()+0xeb) [0x7f68e63f309b]
11: (_start()+0x2a) [0x556effa6fdaa]

*** Caught signal (Aborted) **
in thread 7f68e5ee6180 thread_name:ceph-kvstore-to
ceph version 14.2.4 (75f4de193b3ea58512f204623e6c5a16e6c1e1ba) nautilus (stable)
1: (()+0x12730) [0x7f68e694c730]
2: (gsignal()+0x10b) [0x7f68e64067bb]
3: (abort()+0x121) [0x7f68e63f1535]
4: (ceph::__ceph_assert_fail(char const*, char const*, int, char const*)+0x1a3) [0x7f68e717365f]
5: (()+0x2807e6) [0x7f68e71737e6]
6: (BitmapAllocator::init_rm_free(unsigned long, unsigned long)+0x733) [0x556effc79163]
7: (BlueStore::_open_alloc()+0x1f5) [0x556effa9ff95]
8: (BlueStore::_open_db_and_around(bool)+0xa2) [0x556effab83f2]
9: (BlueStore::_mount(bool, bool)+0x58a) [0x556effafe62a]
10: (StoreTool::load_bluestore(std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > const&, bool)+0x42) [0x556effa6ff22]
11: (StoreTool::StoreTool(std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > const&, std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > const&, bool, bool)+0x226) [0x556effa72f56]
12: (main()+0x28e) [0x556effa4ae4e]
13: (__libc_start_main()+0xeb) [0x7f68e63f309b]
14: (_start()+0x2a) [0x556effa6fdaa]
2019-10-28 16:18:58.793 7f68e5ee6180 -1 *** Caught signal (Aborted) **
in thread 7f68e5ee6180 thread_name:ceph-kvstore-to

ceph version 14.2.4 (75f4de193b3ea58512f204623e6c5a16e6c1e1ba) nautilus (stable)
1: (()+0x12730) [0x7f68e694c730]
2: (gsignal()+0x10b) [0x7f68e64067bb]
3: (abort()+0x121) [0x7f68e63f1535]
4: (ceph::__ceph_assert_fail(char const*, char const*, int, char const*)+0x1a3) [0x7f68e717365f]
5: (()+0x2807e6) [0x7f68e71737e6]
6: (BitmapAllocator::init_rm_free(unsigned long, unsigned long)+0x733) [0x556effc79163]
7: (BlueStore::_open_alloc()+0x1f5) [0x556effa9ff95]
8: (BlueStore::_open_db_and_around(bool)+0xa2) [0x556effab83f2]
9: (BlueStore::_mount(bool, bool)+0x58a) [0x556effafe62a]
10: (StoreTool::load_bluestore(std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > const&, bool)+0x42) [0x556effa6ff22]
11: (StoreTool::StoreTool(std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > const&, std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > const&, bool, bool)+0x226) [0x556effa72f56]
12: (main()+0x28e) [0x556effa4ae4e]
13: (__libc_start_main()+0xeb) [0x7f68e63f309b]
14: (_start()+0x2a) [0x556effa6fdaa]
NOTE: a copy of the executable, or `objdump -rdS <executable>` is needed to interpret this.

--- begin dump of recent events ---
-68> 2019-10-28 16:18:58.565 7f68e5ee6180 5 asok(0x556f02792000) register_command assert hook 0x556f025a8090
-67> 2019-10-28 16:18:58.565 7f68e5ee6180 5 asok(0x556f02792000) register_command abort hook 0x556f025a8090
-66> 2019-10-28 16:18:58.565 7f68e5ee6180 5 asok(0x556f02792000) register_command perfcounters_dump hook 0x556f025a8090
-65> 2019-10-28 16:18:58.565 7f68e5ee6180 5 asok(0x556f02792000) register_command 1 hook 0x556f025a8090
-64> 2019-10-28 16:18:58.565 7f68e5ee6180 5 asok(0x556f02792000) register_command perf dump hook 0x556f025a8090
-63> 2019-10-28 16:18:58.565 7f68e5ee6180 5 asok(0x556f02792000) register_command perfcounters_schema hook 0x556f025a8090
-62> 2019-10-28 16:18:58.565 7f68e5ee6180 5 asok(0x556f02792000) register_command perf histogram dump hook 0x556f025a8090
-61> 2019-10-28 16:18:58.565 7f68e5ee6180 5 asok(0x556f02792000) register_command 2 hook 0x556f025a8090
-60> 2019-10-28 16:18:58.565 7f68e5ee6180 5 asok(0x556f02792000) register_command perf schema hook 0x556f025a8090
-59> 2019-10-28 16:18:58.565 7f68e5ee6180 5 asok(0x556f02792000) register_command perf histogram schema hook 0x556f025a8090
-58> 2019-10-28 16:18:58.565 7f68e5ee6180 5 asok(0x556f02792000) register_command perf reset hook 0x556f025a8090
-57> 2019-10-28 16:18:58.565 7f68e5ee6180 5 asok(0x556f02792000) register_command config show hook 0x556f025a8090
-56> 2019-10-28 16:18:58.565 7f68e5ee6180 5 asok(0x556f02792000) register_command config help hook 0x556f025a8090
-55> 2019-10-28 16:18:58.565 7f68e5ee6180 5 asok(0x556f02792000) register_command config set hook 0x556f025a8090
-54> 2019-10-28 16:18:58.565 7f68e5ee6180 5 asok(0x556f02792000) register_command config unset hook 0x556f025a8090
-53> 2019-10-28 16:18:58.565 7f68e5ee6180 5 asok(0x556f02792000) register_command config get hook 0x556f025a8090
-52> 2019-10-28 16:18:58.565 7f68e5ee6180 5 asok(0x556f02792000) register_command config diff hook 0x556f025a8090
-51> 2019-10-28 16:18:58.565 7f68e5ee6180 5 asok(0x556f02792000) register_command config diff get hook 0x556f025a8090
-50> 2019-10-28 16:18:58.565 7f68e5ee6180 5 asok(0x556f02792000) register_command log flush hook 0x556f025a8090
-49> 2019-10-28 16:18:58.565 7f68e5ee6180 5 asok(0x556f02792000) register_command log dump hook 0x556f025a8090
-48> 2019-10-28 16:18:58.565 7f68e5ee6180 5 asok(0x556f02792000) register_command log reopen hook 0x556f025a8090
-47> 2019-10-28 16:18:58.565 7f68e5ee6180 5 asok(0x556f02792000) register_command dump_mempools hook 0x556f02688068
-46> 2019-10-28 16:18:58.573 7f68e5ee6180 1 bluestore(/var/lib/ceph/osd/ceph-1/) _mount path /var/lib/ceph/osd/ceph-1/
-45> 2019-10-28 16:18:58.573 7f68e5ee6180 1 bdev create path /var/lib/ceph/osd/ceph-1//block type kernel
-44> 2019-10-28 16:18:58.573 7f68e5ee6180 1 bdev(0x556f0325c700 /var/lib/ceph/osd/ceph-1//block) open path /var/lib/ceph/osd/ceph-1//block
-43> 2019-10-28 16:18:58.573 7f68e5ee6180 1 bdev(0x556f0325c700 /var/lib/ceph/osd/ceph-1//block) open size 299964039168 (0x45d7400000, 279 GiB) block_size 4096 (4 KiB) rotational discard not supported
-42> 2019-10-28 16:18:58.573 7f68e5ee6180 1 bluestore(/var/lib/ceph/osd/ceph-1/) _set_cache_sizes cache_size 1073741824 meta 0.4 kv 0.4 data 0.2
-41> 2019-10-28 16:18:58.573 7f68e5ee6180 1 bdev create path /var/lib/ceph/osd/ceph-1//block.db type kernel
-40> 2019-10-28 16:18:58.573 7f68e5ee6180 1 bdev(0x556f0325ce00 /var/lib/ceph/osd/ceph-1//block.db) open path /var/lib/ceph/osd/ceph-1//block.db
-39> 2019-10-28 16:18:58.573 7f68e5ee6180 1 bdev(0x556f0325ce00 /var/lib/ceph/osd/ceph-1//block.db) open size 28991029248 (0x6c0000000, 27 GiB) block_size 4096 (4 KiB) non-rotational discard not supported
-38> 2019-10-28 16:18:58.573 7f68e5ee6180 1 bluefs add_block_device bdev 1 path /var/lib/ceph/osd/ceph-1//block.db size 27 GiB
-37> 2019-10-28 16:18:58.573 7f68e5ee6180 1 bdev create path /var/lib/ceph/osd/ceph-1//block type kernel
-36> 2019-10-28 16:18:58.573 7f68e5ee6180 1 bdev(0x556f0325d180 /var/lib/ceph/osd/ceph-1//block) open path /var/lib/ceph/osd/ceph-1//block
-35> 2019-10-28 16:18:58.573 7f68e5ee6180 1 bdev(0x556f0325d180 /var/lib/ceph/osd/ceph-1//block) open size 299964039168 (0x45d7400000, 279 GiB) block_size 4096 (4 KiB) rotational discard not supported
-34> 2019-10-28 16:18:58.573 7f68e5ee6180 1 bluefs add_block_device bdev 2 path /var/lib/ceph/osd/ceph-1//block size 279 GiB
-33> 2019-10-28 16:18:58.573 7f68e5ee6180 1 bdev create path /var/lib/ceph/osd/ceph-1//block.wal type kernel
-32> 2019-10-28 16:18:58.573 7f68e5ee6180 1 bdev(0x556f0325d500 /var/lib/ceph/osd/ceph-1//block.wal) open path /var/lib/ceph/osd/ceph-1//block.wal
-31> 2019-10-28 16:18:58.573 7f68e5ee6180 1 bdev(0x556f0325d500 /var/lib/ceph/osd/ceph-1//block.wal) open size 1073741824 (0x40000000, 1 GiB) block_size 4096 (4 KiB) non-rotational discard not supported
-30> 2019-10-28 16:18:58.573 7f68e5ee6180 1 bluefs add_block_device bdev 0 path /var/lib/ceph/osd/ceph-1//block.wal size 1 GiB
-29> 2019-10-28 16:18:58.573 7f68e5ee6180 1 bluefs mount
-28> 2019-10-28 16:18:58.581 7f68e5ee6180 0 set rocksdb option compaction_readahead_size = 2097152
-27> 2019-10-28 16:18:58.581 7f68e5ee6180 0 set rocksdb option compression = kNoCompression
-26> 2019-10-28 16:18:58.581 7f68e5ee6180 0 set rocksdb option max_background_compactions = 2
-25> 2019-10-28 16:18:58.581 7f68e5ee6180 0 set rocksdb option max_write_buffer_number = 4
-24> 2019-10-28 16:18:58.581 7f68e5ee6180 0 set rocksdb option min_write_buffer_number_to_merge = 1
-23> 2019-10-28 16:18:58.581 7f68e5ee6180 0 set rocksdb option recycle_log_file_num = 4
-22> 2019-10-28 16:18:58.581 7f68e5ee6180 0 set rocksdb option writable_file_max_buffer_size = 0
-21> 2019-10-28 16:18:58.581 7f68e5ee6180 0 set rocksdb option write_buffer_size = 268435456
-20> 2019-10-28 16:18:58.581 7f68e5ee6180 0 set rocksdb option compaction_readahead_size = 2097152
-19> 2019-10-28 16:18:58.581 7f68e5ee6180 0 set rocksdb option compression = kNoCompression
-18> 2019-10-28 16:18:58.581 7f68e5ee6180 0 set rocksdb option max_background_compactions = 2
-17> 2019-10-28 16:18:58.581 7f68e5ee6180 0 set rocksdb option max_write_buffer_number = 4
-16> 2019-10-28 16:18:58.581 7f68e5ee6180 0 set rocksdb option min_write_buffer_number_to_merge = 1
-15> 2019-10-28 16:18:58.581 7f68e5ee6180 0 set rocksdb option recycle_log_file_num = 4
-14> 2019-10-28 16:18:58.581 7f68e5ee6180 0 set rocksdb option writable_file_max_buffer_size = 0
-13> 2019-10-28 16:18:58.581 7f68e5ee6180 0 set rocksdb option write_buffer_size = 268435456
-12> 2019-10-28 16:18:58.589 7f68e5ee6180 1 rocksdb: do_open column families: [default]
-11> 2019-10-28 16:18:58.777 7f68e5ee6180 1 bluestore(/var/lib/ceph/osd/ceph-1/) _open_db opened rocksdb path db options compression=kNoCompression,max_write_buffer_number=4,min_write_buffer_number_to_merge=1,recycle_log_file_num=4,write_buffer_size=268435456,writable_file_max_buffer_size=0,compaction_readahead_size=2097152,max_background_compactions=2
-10> 2019-10-28 16:18:58.785 7f68e5ee6180 1 get compressor snappy = 0
-9> 2019-10-28 16:18:58.785 7f68e5ee6180 1 load compressor snappy
-8> 2019-10-28 16:18:58.785 7f68e5ee6180 1 add compressor snappy 0x556f02668e70
-7> 2019-10-28 16:18:58.785 7f68e5ee6180 1 get compressor snappy = 0x556f02668e70
-6> 2019-10-28 16:18:58.785 7f68e5ee6180 1 load: compressor snappy loaded and registered
-5> 2019-10-28 16:18:58.785 7f68e5ee6180 1 get compressor snappy = 0x556f02668e70
-4> 2019-10-28 16:18:58.785 7f68e5ee6180 1 freelist init
-3> 2019-10-28 16:18:58.789 7f68e5ee6180 1 bluestore(/var/lib/ceph/osd/ceph-1/) _open_alloc opening allocation metadata
-2> 2019-10-28 16:18:58.789 7f68e5ee6180 1 bluestore(/var/lib/ceph/osd/ceph-1/) _open_alloc loaded 0 B in 0 extents
-1> 2019-10-28 16:18:58.793 7f68e5ee6180 -1 /data/Debian/WORKDIR/ceph-14.2.4/src/os/bluestore/fastbmap_allocator_impl.h: In function 'void AllocatorLevel02<T>::_mark_allocated(uint64_t, uint64_t) [with L1 = AllocatorLevel01Loose; uint64_t = long unsigned int]' thread 7f68e5ee6180 time 2019-10-28 16:18:58.794917
/data/Debian/WORKDIR/ceph-14.2.4/src/os/bluestore/fastbmap_allocator_impl.h: 750: FAILED ceph_assert(available >= allocated)

ceph version 14.2.4 (75f4de193b3ea58512f204623e6c5a16e6c1e1ba) nautilus (stable)
1: (ceph::__ceph_assert_fail(char const*, char const*, int, char const*)+0x152) [0x7f68e717360e]
2: (()+0x2807e6) [0x7f68e71737e6]
3: (BitmapAllocator::init_rm_free(unsigned long, unsigned long)+0x733) [0x556effc79163]
4: (BlueStore::_open_alloc()+0x1f5) [0x556effa9ff95]
5: (BlueStore::_open_db_and_around(bool)+0xa2) [0x556effab83f2]
6: (BlueStore::_mount(bool, bool)+0x58a) [0x556effafe62a]
7: (StoreTool::load_bluestore(std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > const&, bool)+0x42) [0x556effa6ff22]
8: (StoreTool::StoreTool(std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > const&, std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > const&, bool, bool)+0x226) [0x556effa72f56]
9: (main()+0x28e) [0x556effa4ae4e]
10: (__libc_start_main()+0xeb) [0x7f68e63f309b]
11: (_start()+0x2a) [0x556effa6fdaa]

0> 2019-10-28 16:18:58.793 7f68e5ee6180 -1 *** Caught signal (Aborted) **
in thread 7f68e5ee6180 thread_name:ceph-kvstore-to

ceph version 14.2.4 (75f4de193b3ea58512f204623e6c5a16e6c1e1ba) nautilus (stable)
1: (()+0x12730) [0x7f68e694c730]
2: (gsignal()+0x10b) [0x7f68e64067bb]
3: (abort()+0x121) [0x7f68e63f1535]
4: (ceph::__ceph_assert_fail(char const*, char const*, int, char const*)+0x1a3) [0x7f68e717365f]
5: (()+0x2807e6) [0x7f68e71737e6]
6: (BitmapAllocator::init_rm_free(unsigned long, unsigned long)+0x733) [0x556effc79163]
7: (BlueStore::_open_alloc()+0x1f5) [0x556effa9ff95]
8: (BlueStore::_open_db_and_around(bool)+0xa2) [0x556effab83f2]
9: (BlueStore::_mount(bool, bool)+0x58a) [0x556effafe62a]
10: (StoreTool::load_bluestore(std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > const&, bool)+0x42) [0x556effa6ff22]
11: (StoreTool::StoreTool(std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > const&, std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > const&, bool, bool)+0x226) [0x556effa72f56]
12: (main()+0x28e) [0x556effa4ae4e]
13: (__libc_start_main()+0xeb) [0x7f68e63f309b]
14: (_start()+0x2a) [0x556effa6fdaa]
NOTE: a copy of the executable, or `objdump -rdS <executable>` is needed to interpret this.

--- logging levels ---
0/ 5 none
0/ 1 lockdep
0/ 1 context
1/ 1 crush
1/ 5 mds
1/ 5 mds_balancer
1/ 5 mds_locker
1/ 5 mds_log
1/ 5 mds_log_expire
1/ 5 mds_migrator
0/ 1 buffer
0/ 1 timer
0/ 1 filer
0/ 1 striper
0/ 1 objecter
0/ 5 rados
0/ 5 rbd
0/ 5 rbd_mirror
0/ 5 rbd_replay
0/ 5 journaler
0/ 5 objectcacher
0/ 5 client
1/ 5 osd
0/ 5 optracker
0/ 5 objclass
1/ 3 filestore
1/ 3 journal
0/ 0 ms
1/ 5 mon
0/10 monc
1/ 5 paxos
0/ 5 tp
1/ 5 auth
1/ 5 crypto
1/ 1 finisher
1/ 1 reserver
1/ 5 heartbeatmap
1/ 5 perfcounter
1/ 5 rgw
1/ 5 rgw_sync
1/10 civetweb
1/ 5 javaclient
1/ 5 asok
1/ 1 throttle
0/ 0 refs
1/ 5 xio
1/ 5 compressor
1/ 5 bluestore
1/ 5 bluefs
1/ 3 bdev
1/ 5 kstore
2/ 2 rocksdb
4/ 5 leveldb
4/ 5 memdb
1/ 5 kinetic
1/ 5 fuse
1/ 5 mgr
1/ 5 mgrc
1/ 5 dpdk
1/ 5 eventtrace
-2/-2 (syslog threshold)
99/99 (stderr threshold)
max_recent 500
max_new 1000
log_file
--- end dump of recent events ---
--- begin dump of recent events ---
-68> 2019-10-28 16:18:58.565 7f68e5ee6180 5 asok(0x556f02792000) register_command assert hook 0x556f025a8090
-67> 2019-10-28 16:18:58.565 7f68e5ee6180 5 asok(0x556f02792000) register_command abort hook 0x556f025a8090
-66> 2019-10-28 16:18:58.565 7f68e5ee6180 5 asok(0x556f02792000) register_command perfcounters_dump hook 0x556f025a8090
-65> 2019-10-28 16:18:58.565 7f68e5ee6180 5 asok(0x556f02792000) register_command 1 hook 0x556f025a8090
-64> 2019-10-28 16:18:58.565 7f68e5ee6180 5 asok(0x556f02792000) register_command perf dump hook 0x556f025a8090
-63> 2019-10-28 16:18:58.565 7f68e5ee6180 5 asok(0x556f02792000) register_command perfcounters_schema hook 0x556f025a8090
-62> 2019-10-28 16:18:58.565 7f68e5ee6180 5 asok(0x556f02792000) register_command perf histogram dump hook 0x556f025a8090
-61> 2019-10-28 16:18:58.565 7f68e5ee6180 5 asok(0x556f02792000) register_command 2 hook 0x556f025a8090
-60> 2019-10-28 16:18:58.565 7f68e5ee6180 5 asok(0x556f02792000) register_command perf schema hook 0x556f025a8090
-59> 2019-10-28 16:18:58.565 7f68e5ee6180 5 asok(0x556f02792000) register_command perf histogram schema hook 0x556f025a8090
-58> 2019-10-28 16:18:58.565 7f68e5ee6180 5 asok(0x556f02792000) register_command perf reset hook 0x556f025a8090
-57> 2019-10-28 16:18:58.565 7f68e5ee6180 5 asok(0x556f02792000) register_command config show hook 0x556f025a8090
-56> 2019-10-28 16:18:58.565 7f68e5ee6180 5 asok(0x556f02792000) register_command config help hook 0x556f025a8090
-55> 2019-10-28 16:18:58.565 7f68e5ee6180 5 asok(0x556f02792000) register_command config set hook 0x556f025a8090
-54> 2019-10-28 16:18:58.565 7f68e5ee6180 5 asok(0x556f02792000) register_command config unset hook 0x556f025a8090
-53> 2019-10-28 16:18:58.565 7f68e5ee6180 5 asok(0x556f02792000) register_command config get hook 0x556f025a8090
-52> 2019-10-28 16:18:58.565 7f68e5ee6180 5 asok(0x556f02792000) register_command config diff hook 0x556f025a8090
-51> 2019-10-28 16:18:58.565 7f68e5ee6180 5 asok(0x556f02792000) register_command config diff get hook 0x556f025a8090
-50> 2019-10-28 16:18:58.565 7f68e5ee6180 5 asok(0x556f02792000) register_command log flush hook 0x556f025a8090
-49> 2019-10-28 16:18:58.565 7f68e5ee6180 5 asok(0x556f02792000) register_command log dump hook 0x556f025a8090
-48> 2019-10-28 16:18:58.565 7f68e5ee6180 5 asok(0x556f02792000) register_command log reopen hook 0x556f025a8090
-47> 2019-10-28 16:18:58.565 7f68e5ee6180 5 asok(0x556f02792000) register_command dump_mempools hook 0x556f02688068
-46> 2019-10-28 16:18:58.573 7f68e5ee6180 1 bluestore(/var/lib/ceph/osd/ceph-1/) _mount path /var/lib/ceph/osd/ceph-1/
-45> 2019-10-28 16:18:58.573 7f68e5ee6180 1 bdev create path /var/lib/ceph/osd/ceph-1//block type kernel
-44> 2019-10-28 16:18:58.573 7f68e5ee6180 1 bdev(0x556f0325c700 /var/lib/ceph/osd/ceph-1//block) open path /var/lib/ceph/osd/ceph-1//block
-43> 2019-10-28 16:18:58.573 7f68e5ee6180 1 bdev(0x556f0325c700 /var/lib/ceph/osd/ceph-1//block) open size 299964039168 (0x45d7400000, 279 GiB) block_size 4096 (4 KiB) rotational discard not supported
-42> 2019-10-28 16:18:58.573 7f68e5ee6180 1 bluestore(/var/lib/ceph/osd/ceph-1/) _set_cache_sizes cache_size 1073741824 meta 0.4 kv 0.4 data 0.2
-41> 2019-10-28 16:18:58.573 7f68e5ee6180 1 bdev create path /var/lib/ceph/osd/ceph-1//block.db type kernel
-40> 2019-10-28 16:18:58.573 7f68e5ee6180 1 bdev(0x556f0325ce00 /var/lib/ceph/osd/ceph-1//block.db) open path /var/lib/ceph/osd/ceph-1//block.db
-39> 2019-10-28 16:18:58.573 7f68e5ee6180 1 bdev(0x556f0325ce00 /var/lib/ceph/osd/ceph-1//block.db) open size 28991029248 (0x6c0000000, 27 GiB) block_size 4096 (4 KiB) non-rotational discard not supported
-38> 2019-10-28 16:18:58.573 7f68e5ee6180 1 bluefs add_block_device bdev 1 path /var/lib/ceph/osd/ceph-1//block.db size 27 GiB
-37> 2019-10-28 16:18:58.573 7f68e5ee6180 1 bdev create path /var/lib/ceph/osd/ceph-1//block type kernel
-36> 2019-10-28 16:18:58.573 7f68e5ee6180 1 bdev(0x556f0325d180 /var/lib/ceph/osd/ceph-1//block) open path /var/lib/ceph/osd/ceph-1//block
-35> 2019-10-28 16:18:58.573 7f68e5ee6180 1 bdev(0x556f0325d180 /var/lib/ceph/osd/ceph-1//block) open size 299964039168 (0x45d7400000, 279 GiB) block_size 4096 (4 KiB) rotational discard not supported
-34> 2019-10-28 16:18:58.573 7f68e5ee6180 1 bluefs add_block_device bdev 2 path /var/lib/ceph/osd/ceph-1//block size 279 GiB
-33> 2019-10-28 16:18:58.573 7f68e5ee6180 1 bdev create path /var/lib/ceph/osd/ceph-1//block.wal type kernel
-32> 2019-10-28 16:18:58.573 7f68e5ee6180 1 bdev(0x556f0325d500 /var/lib/ceph/osd/ceph-1//block.wal) open path /var/lib/ceph/osd/ceph-1//block.wal
-31> 2019-10-28 16:18:58.573 7f68e5ee6180 1 bdev(0x556f0325d500 /var/lib/ceph/osd/ceph-1//block.wal) open size 1073741824 (0x40000000, 1 GiB) block_size 4096 (4 KiB) non-rotational discard not supported
-30> 2019-10-28 16:18:58.573 7f68e5ee6180 1 bluefs add_block_device bdev 0 path /var/lib/ceph/osd/ceph-1//block.wal size 1 GiB
-29> 2019-10-28 16:18:58.573 7f68e5ee6180 1 bluefs mount
-28> 2019-10-28 16:18:58.581 7f68e5ee6180 0 set rocksdb option compaction_readahead_size = 2097152
-27> 2019-10-28 16:18:58.581 7f68e5ee6180 0 set rocksdb option compression = kNoCompression
-26> 2019-10-28 16:18:58.581 7f68e5ee6180 0 set rocksdb option max_background_compactions = 2
-25> 2019-10-28 16:18:58.581 7f68e5ee6180 0 set rocksdb option max_write_buffer_number = 4
-24> 2019-10-28 16:18:58.581 7f68e5ee6180 0 set rocksdb option min_write_buffer_number_to_merge = 1
-23> 2019-10-28 16:18:58.581 7f68e5ee6180 0 set rocksdb option recycle_log_file_num = 4
-22> 2019-10-28 16:18:58.581 7f68e5ee6180 0 set rocksdb option writable_file_max_buffer_size = 0
-21> 2019-10-28 16:18:58.581 7f68e5ee6180 0 set rocksdb option write_buffer_size = 268435456
-20> 2019-10-28 16:18:58.581 7f68e5ee6180 0 set rocksdb option compaction_readahead_size = 2097152
-19> 2019-10-28 16:18:58.581 7f68e5ee6180 0 set rocksdb option compression = kNoCompression
-18> 2019-10-28 16:18:58.581 7f68e5ee6180 0 set rocksdb option max_background_compactions = 2
-17> 2019-10-28 16:18:58.581 7f68e5ee6180 0 set rocksdb option max_write_buffer_number = 4
-16> 2019-10-28 16:18:58.581 7f68e5ee6180 0 set rocksdb option min_write_buffer_number_to_merge = 1
-15> 2019-10-28 16:18:58.581 7f68e5ee6180 0 set rocksdb option recycle_log_file_num = 4
-14> 2019-10-28 16:18:58.581 7f68e5ee6180 0 set rocksdb option writable_file_max_buffer_size = 0
-13> 2019-10-28 16:18:58.581 7f68e5ee6180 0 set rocksdb option write_buffer_size = 268435456
-12> 2019-10-28 16:18:58.589 7f68e5ee6180 1 rocksdb: do_open column families: [default]
-11> 2019-10-28 16:18:58.777 7f68e5ee6180 1 bluestore(/var/lib/ceph/osd/ceph-1/) _open_db opened rocksdb path db options compression=kNoCompression,max_write_buffer_number=4,min_write_buffer_number_to_merge=1,recycle_log_file_num=4,write_buffer_size=268435456,writable_file_max_buffer_size=0,compaction_readahead_size=2097152,max_background_compactions=2
-10> 2019-10-28 16:18:58.785 7f68e5ee6180 1 get compressor snappy = 0
-9> 2019-10-28 16:18:58.785 7f68e5ee6180 1 load compressor snappy
-8> 2019-10-28 16:18:58.785 7f68e5ee6180 1 add compressor snappy 0x556f02668e70
-7> 2019-10-28 16:18:58.785 7f68e5ee6180 1 get compressor snappy = 0x556f02668e70
-6> 2019-10-28 16:18:58.785 7f68e5ee6180 1 load: compressor snappy loaded and registered
-5> 2019-10-28 16:18:58.785 7f68e5ee6180 1 get compressor snappy = 0x556f02668e70
-4> 2019-10-28 16:18:58.785 7f68e5ee6180 1 freelist init
-3> 2019-10-28 16:18:58.789 7f68e5ee6180 1 bluestore(/var/lib/ceph/osd/ceph-1/) _open_alloc opening allocation metadata
-2> 2019-10-28 16:18:58.789 7f68e5ee6180 1 bluestore(/var/lib/ceph/osd/ceph-1/) _open_alloc loaded 0 B in 0 extents
-1> 2019-10-28 16:18:58.793 7f68e5ee6180 -1 /data/Debian/WORKDIR/ceph-14.2.4/src/os/bluestore/fastbmap_allocator_impl.h: In function 'void AllocatorLevel02<T>::_mark_allocated(uint64_t, uint64_t) [with L1 = AllocatorLevel01Loose; uint64_t = long unsigned int]' thread 7f68e5ee6180 time 2019-10-28 16:18:58.794917
/data/Debian/WORKDIR/ceph-14.2.4/src/os/bluestore/fastbmap_allocator_impl.h: 750: FAILED ceph_assert(available >= allocated)

ceph version 14.2.4 (75f4de193b3ea58512f204623e6c5a16e6c1e1ba) nautilus (stable)
1: (ceph::__ceph_assert_fail(char const*, char const*, int, char const*)+0x152) [0x7f68e717360e]
2: (()+0x2807e6) [0x7f68e71737e6]
3: (BitmapAllocator::init_rm_free(unsigned long, unsigned long)+0x733) [0x556effc79163]
4: (BlueStore::_open_alloc()+0x1f5) [0x556effa9ff95]
5: (BlueStore::_open_db_and_around(bool)+0xa2) [0x556effab83f2]
6: (BlueStore::_mount(bool, bool)+0x58a) [0x556effafe62a]
7: (StoreTool::load_bluestore(std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > const&, bool)+0x42) [0x556effa6ff22]
8: (StoreTool::StoreTool(std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > const&, std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > const&, bool, bool)+0x226) [0x556effa72f56]
9: (main()+0x28e) [0x556effa4ae4e]
10: (__libc_start_main()+0xeb) [0x7f68e63f309b]
11: (_start()+0x2a) [0x556effa6fdaa]

0> 2019-10-28 16:18:58.793 7f68e5ee6180 -1 *** Caught signal (Aborted) **
in thread 7f68e5ee6180 thread_name:ceph-kvstore-to

ceph version 14.2.4 (75f4de193b3ea58512f204623e6c5a16e6c1e1ba) nautilus (stable)
1: (()+0x12730) [0x7f68e694c730]
2: (gsignal()+0x10b) [0x7f68e64067bb]
3: (abort()+0x121) [0x7f68e63f1535]
4: (ceph::__ceph_assert_fail(char const*, char const*, int, char const*)+0x1a3) [0x7f68e717365f]
5: (()+0x2807e6) [0x7f68e71737e6]
6: (BitmapAllocator::init_rm_free(unsigned long, unsigned long)+0x733) [0x556effc79163]
7: (BlueStore::_open_alloc()+0x1f5) [0x556effa9ff95]
8: (BlueStore::_open_db_and_around(bool)+0xa2) [0x556effab83f2]
9: (BlueStore::_mount(bool, bool)+0x58a) [0x556effafe62a]
10: (StoreTool::load_bluestore(std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > const&, bool)+0x42) [0x556effa6ff22]
11: (StoreTool::StoreTool(std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > const&, std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > const&, bool, bool)+0x226) [0x556effa72f56]
12: (main()+0x28e) [0x556effa4ae4e]
13: (__libc_start_main()+0xeb) [0x7f68e63f309b]
14: (_start()+0x2a) [0x556effa6fdaa]
NOTE: a copy of the executable, or `objdump -rdS <executable>` is needed to interpret this.

--- logging levels ---
0/ 5 none
0/ 1 lockdep
0/ 1 context
1/ 1 crush
1/ 5 mds
1/ 5 mds_balancer
1/ 5 mds_locker
1/ 5 mds_log
1/ 5 mds_log_expire
1/ 5 mds_migrator
0/ 1 buffer
0/ 1 timer
0/ 1 filer
0/ 1 striper
0/ 1 objecter
0/ 5 rados
0/ 5 rbd
0/ 5 rbd_mirror
0/ 5 rbd_replay
0/ 5 journaler
0/ 5 objectcacher
0/ 5 client
1/ 5 osd
0/ 5 optracker
0/ 5 objclass
1/ 3 filestore
1/ 3 journal
0/ 0 ms
1/ 5 mon
0/10 monc
1/ 5 paxos
0/ 5 tp
1/ 5 auth
1/ 5 crypto
1/ 1 finisher
1/ 1 reserver
1/ 5 heartbeatmap
1/ 5 perfcounter
1/ 5 rgw
1/ 5 rgw_sync
1/10 civetweb
1/ 5 javaclient
1/ 5 asok
1/ 1 throttle
0/ 0 refs
1/ 5 xio
1/ 5 compressor
1/ 5 bluestore
1/ 5 bluefs
1/ 3 bdev
1/ 5 kstore
2/ 2 rocksdb
4/ 5 leveldb
4/ 5 memdb
1/ 5 kinetic
1/ 5 fuse
1/ 5 mgr
1/ 5 mgrc
1/ 5 dpdk
1/ 5 eventtrace
-2/-2 (syslog threshold)
99/99 (stderr threshold)
max_recent 500
max_new 1000
log_file /var/lib/ceph/crash/2019-10-28_15:18:58.797756Z_4a79940e-ee55-4222-987b-1fa6aa2fcbd4/log
--- end dump of recent events ---
(11-11/31)