Actions
Bug #41211
openosd crash due to rocksdb
Status:
Need More Info
Priority:
Normal
Assignee:
-
Category:
-
Target version:
-
% Done:
0%
Source:
Community (user)
Tags:
Backport:
Regression:
No
Severity:
3 - minor
Reviewed:
Affected Versions:
ceph-qa-suite:
Component(RADOS):
Pull request ID:
Crash signature (v1):
Crash signature (v2):
Description
Reported in "[ceph-users] Possibly a bug on rocksdb" on 12.2.12
[root@horeb72 ceph]# head -400 ceph-osd.4.log
2019-08-11 07:30:02.186519 7f69bd020700 0 -- 192.168.10.72:6805/5915 >> 192.168.10.73:6801/4096 conn(0x56549cfc0800 :6805 s=STATE_ACCEPTING_WAIT_CONNECT_MSG_AUTH pgs=0 cs=0 l=0).handle_connect_msg accept connect_seq 15 vs existing csq=15 existing_state=STATE_STANDBY
2019-08-11 07:30:02.186871 7f69bd020700 0 -- 192.168.10.72:6805/5915 >> 192.168.10.73:6801/4096 conn(0x56549cfc0800 :6805 s=STATE_ACCEPTING_WAIT_CONNECT_MSG_AUTH pgs=0 cs=0 l=0).handle_connect_msg accept connect_seq 16 vs existing csq=15 existing_state=STATE_STANDBY
2019-08-11 07:30:02.242291 7f69bc81f700 0 -- 192.168.10.72:6805/5915 >> 192.168.10.71:6805/5046 conn(0x5654b93ed000 :6805 s=STATE_ACCEPTING_WAIT_CONNECT_MSG_AUTH pgs=0 cs=0 l=0).handle_connect_msg accept connect_seq 15 vs existing csq=15 existing_state=STATE_STANDBY
2019-08-11 07:30:02.242554 7f69bc81f700 0 -- 192.168.10.72:6805/5915 >> 192.168.10.71:6805/5046 conn(0x5654b93ed000 :6805 s=STATE_ACCEPTING_WAIT_CONNECT_MSG_AUTH pgs=0 cs=0 l=0).handle_connect_msg accept connect_seq 16 vs existing csq=15 existing_state=STATE_STANDBY
2019-08-11 07:30:02.260295 7f69bc81f700 0 -- 192.168.10.72:6805/5915 >> 192.168.10.73:6806/4864 conn(0x56544de16800 :6805 s=STATE_ACCEPTING_WAIT_CONNECT_MSG_AUTH pgs=0 cs=0 l=0).handle_connect_msg accept connect_seq 15 vs existing csq=15 existing_state=STATE_CONNECTING_WAIT_CONNECT_REPLY
2019-08-11 17:11:01.968247 7ff4822f1d80 -1 WARNING: the following dangerous and experimental features are enabled: bluestore,rocksdb
2019-08-11 17:11:01.968333 7ff4822f1d80 0 ceph version 12.2.12 (1436006594665279fe734b4c15d7e08c13ebd777) luminous (stable), process ceph-osd, pid 1048682
2019-08-11 17:11:01.970611 7ff4822f1d80 0 pidfile_write: ignore empty --pid-file
2019-08-11 17:11:01.991542 7ff4822f1d80 -1 WARNING: the following dangerous and experimental features are enabled: bluestore,rocksdb
2019-08-11 17:11:01.997597 7ff4822f1d80 0 load: jerasure load: lrc load: isa
2019-08-11 17:11:01.997710 7ff4822f1d80 1 bdev create path /var/lib/ceph/osd/ceph-4/block type kernel
2019-08-11 17:11:01.997723 7ff4822f1d80 1 bdev(0x564774656c00 /var/lib/ceph/osd/ceph-4/block) open path /var/lib/ceph/osd/ceph-4/block
2019-08-11 17:11:01.998127 7ff4822f1d80 1 bdev(0x564774656c00 /var/lib/ceph/osd/ceph-4/block) open size 858887553024 (0xc7f9b00000, 800GiB) block_size 4096 (4KiB) non-rotational
2019-08-11 17:11:01.998231 7ff4822f1d80 1 bdev(0x564774656c00 /var/lib/ceph/osd/ceph-4/block) close
2019-08-11 17:11:02.265144 7ff4822f1d80 1 bdev create path /var/lib/ceph/osd/ceph-4/block type kernel
2019-08-11 17:11:02.265177 7ff4822f1d80 1 bdev(0x564774658a00 /var/lib/ceph/osd/ceph-4/block) open path /var/lib/ceph/osd/ceph-4/block
2019-08-11 17:11:02.265695 7ff4822f1d80 1 bdev(0x564774658a00 /var/lib/ceph/osd/ceph-4/block) open size 858887553024 (0xc7f9b00000, 800GiB) block_size 4096 (4KiB) non-rotational
2019-08-11 17:11:02.266233 7ff4822f1d80 1 bdev create path /var/lib/ceph/osd/ceph-4/block.db type kernel
2019-08-11 17:11:02.266256 7ff4822f1d80 1 bdev(0x564774589a00 /var/lib/ceph/osd/ceph-4/block.db) open path /var/lib/ceph/osd/ceph-4/block.db
2019-08-11 17:11:02.266812 7ff4822f1d80 1 bdev(0x564774589a00 /var/lib/ceph/osd/ceph-4/block.db) open size 29999759360 (0x6fc200000, 27.9GiB) block_size 4096 (4KiB) non-rotational
2019-08-11 17:11:02.266998 7ff4822f1d80 1 bdev create path /var/lib/ceph/osd/ceph-4/block type kernel
2019-08-11 17:11:02.267015 7ff4822f1d80 1 bdev(0x564774659a00 /var/lib/ceph/osd/ceph-4/block) open path /var/lib/ceph/osd/ceph-4/block
2019-08-11 17:11:02.267412 7ff4822f1d80 1 bdev(0x564774659a00 /var/lib/ceph/osd/ceph-4/block) open size 858887553024 (0xc7f9b00000, 800GiB) block_size 4096 (4KiB) non-rotational
2019-08-11 17:11:02.298355 7ff4822f1d80 0 set rocksdb option compaction_readahead_size = 2MB
2019-08-11 17:11:02.298368 7ff4822f1d80 0 set rocksdb option compaction_style = kCompactionStyleLevel
2019-08-11 17:11:02.299628 7ff4822f1d80 0 set rocksdb option compaction_threads = 32
2019-08-11 17:11:02.299648 7ff4822f1d80 0 set rocksdb option compression = kNoCompression
2019-08-11 17:11:02.299993 7ff4822f1d80 0 set rocksdb option flusher_threads = 8
2019-08-11 17:11:02.300006 7ff4822f1d80 0 set rocksdb option level0_file_num_compaction_trigger = 64
2019-08-11 17:11:02.300011 7ff4822f1d80 0 set rocksdb option level0_slowdown_writes_trigger = 128
2019-08-11 17:11:02.300017 7ff4822f1d80 0 set rocksdb option level0_stop_writes_trigger = 256
2019-08-11 17:11:02.300022 7ff4822f1d80 0 set rocksdb option max_background_compactions = 64
2019-08-11 17:11:02.300027 7ff4822f1d80 0 set rocksdb option max_bytes_for_level_base = 2GB
2019-08-11 17:11:02.300034 7ff4822f1d80 0 set rocksdb option max_write_buffer_number = 64
2019-08-11 17:11:02.300039 7ff4822f1d80 0 set rocksdb option min_write_buffer_number_to_merge = 32
2019-08-11 17:11:02.300044 7ff4822f1d80 0 set rocksdb option recycle_log_file_num = 64
2019-08-11 17:11:02.300048 7ff4822f1d80 0 set rocksdb option target_file_size_base = 4MB
2019-08-11 17:11:02.300053 7ff4822f1d80 0 set rocksdb option write_buffer_size = 4MB
2019-08-11 17:11:02.300093 7ff4822f1d80 0 set rocksdb option compaction_readahead_size = 2MB
2019-08-11 17:11:02.300103 7ff4822f1d80 0 set rocksdb option compaction_style = kCompactionStyleLevel
2019-08-11 17:11:02.300110 7ff4822f1d80 0 set rocksdb option compaction_threads = 32
2019-08-11 17:11:02.300121 7ff4822f1d80 0 set rocksdb option compression = kNoCompression
2019-08-11 17:11:02.300129 7ff4822f1d80 0 set rocksdb option flusher_threads = 8
2019-08-11 17:11:02.300135 7ff4822f1d80 0 set rocksdb option level0_file_num_compaction_trigger = 64
2019-08-11 17:11:02.300142 7ff4822f1d80 0 set rocksdb option level0_slowdown_writes_trigger = 128
2019-08-11 17:11:02.300146 7ff4822f1d80 0 set rocksdb option level0_stop_writes_trigger = 256
2019-08-11 17:11:02.300150 7ff4822f1d80 0 set rocksdb option max_background_compactions = 64
2019-08-11 17:11:02.300155 7ff4822f1d80 0 set rocksdb option max_bytes_for_level_base = 2GB
2019-08-11 17:11:02.300159 7ff4822f1d80 0 set rocksdb option max_write_buffer_number = 64
2019-08-11 17:11:02.300166 7ff4822f1d80 0 set rocksdb option min_write_buffer_number_to_merge = 32
2019-08-11 17:11:02.300176 7ff4822f1d80 0 set rocksdb option recycle_log_file_num = 64
2019-08-11 17:11:02.300185 7ff4822f1d80 0 set rocksdb option target_file_size_base = 4MB
2019-08-11 17:11:02.300193 7ff4822f1d80 0 set rocksdb option write_buffer_size = 4MB
2019-08-11 17:11:02.819067 7ff4822f1d80 -1 *** Caught signal (Aborted) **
in thread 7ff4822f1d80 thread_name:ceph-osd
ceph version 12.2.12 (1436006594665279fe734b4c15d7e08c13ebd777) luminous (stable)
1: (()+0xa64ee1) [0x56476aafbee1]
2: (()+0xf6d0) [0x7ff47f5a16d0]
3: (gsignal()+0x37) [0x7ff47e5c2277]
4: (abort()+0x148) [0x7ff47e5c3968]
5: (__gnu_cxx::__verbose_terminate_handler()+0x165) [0x7ff47eed17d5]
6: (()+0x5e746) [0x7ff47eecf746]
7: (()+0x5e773) [0x7ff47eecf773]
8: (()+0x5e993) [0x7ff47eecf993]
9: (()+0xa6f149) [0x56476ab06149]
10: (decode(std::string&, ceph::buffer::list::iterator&)+0x53) [0x56476a765313]
11: (OSDSuperblock::decode(ceph::buffer::list::iterator&)+0x70) [0x56476a7f23e0]
12: (OSD::read_superblock()+0x193) [0x56476a567943]
13: (OSD::init()+0x773) [0x56476a5b4eb3]
14: (main()+0x2d07) [0x56476a4b7ef7]
15: (__libc_start_main()+0xf5) [0x7ff47e5ae445]
16: (()+0x4c0dc3) [0x56476a557dc3]
NOTE: a copy of the executable, or `objdump -rdS <executable>` is needed to interpret this.
--- begin dump of recent events ---
-68> 2019-08-11 17:11:01.963739 7ff4822f1d80 5 asok(0x56477431e1c0) register_command perfcounters_dump hook 0x5647742e4180
-67> 2019-08-11 17:11:01.963761 7ff4822f1d80 5 asok(0x56477431e1c0) register_command 1 hook 0x5647742e4180
-66> 2019-08-11 17:11:01.963765 7ff4822f1d80 5 asok(0x56477431e1c0) register_command perf dump hook 0x5647742e4180
-65> 2019-08-11 17:11:01.963767 7ff4822f1d80 5 asok(0x56477431e1c0) register_command perfcounters_schema hook 0x5647742e4180
-64> 2019-08-11 17:11:01.963772 7ff4822f1d80 5 asok(0x56477431e1c0) register_command perf histogram dump hook 0x5647742e4180
-63> 2019-08-11 17:11:01.963775 7ff4822f1d80 5 asok(0x56477431e1c0) register_command 2 hook 0x5647742e4180
-62> 2019-08-11 17:11:01.963777 7ff4822f1d80 5 asok(0x56477431e1c0) register_command perf schema hook 0x5647742e4180
-61> 2019-08-11 17:11:01.963780 7ff4822f1d80 5 asok(0x56477431e1c0) register_command perf histogram schema hook 0x5647742e4180
-60> 2019-08-11 17:11:01.963788 7ff4822f1d80 5 asok(0x56477431e1c0) register_command perf reset hook 0x5647742e4180
-59> 2019-08-11 17:11:01.963792 7ff4822f1d80 5 asok(0x56477431e1c0) register_command config show hook 0x5647742e4180
-58> 2019-08-11 17:11:01.963795 7ff4822f1d80 5 asok(0x56477431e1c0) register_command config help hook 0x5647742e4180
-57> 2019-08-11 17:11:01.963799 7ff4822f1d80 5 asok(0x56477431e1c0) register_command config set hook 0x5647742e4180
-56> 2019-08-11 17:11:01.963803 7ff4822f1d80 5 asok(0x56477431e1c0) register_command config get hook 0x5647742e4180
-55> 2019-08-11 17:11:01.963806 7ff4822f1d80 5 asok(0x56477431e1c0) register_command config diff hook 0x5647742e4180
-54> 2019-08-11 17:11:01.963809 7ff4822f1d80 5 asok(0x56477431e1c0) register_command config diff get hook 0x5647742e4180
-53> 2019-08-11 17:11:01.963812 7ff4822f1d80 5 asok(0x56477431e1c0) register_command log flush hook 0x5647742e4180
-52> 2019-08-11 17:11:01.963815 7ff4822f1d80 5 asok(0x56477431e1c0) register_command log dump hook 0x5647742e4180
-51> 2019-08-11 17:11:01.963818 7ff4822f1d80 5 asok(0x56477431e1c0) register_command log reopen hook 0x5647742e4180
-50> 2019-08-11 17:11:01.963827 7ff4822f1d80 5 asok(0x56477431e1c0) register_command dump_mempools hook 0x564774307ea8
-49> 2019-08-11 17:11:01.968040 7ff4822f1d80 -1 WARNING: the following dangerous and experimental features are enabled: bluestore,rocksdb
-48> 2019-08-11 17:11:01.968247 7ff4822f1d80 -1 WARNING: the following dangerous and experimental features are enabled: bluestore,rocksdb
-47> 2019-08-11 17:11:01.968333 7ff4822f1d80 0 ceph version 12.2.12 (1436006594665279fe734b4c15d7e08c13ebd777) luminous (stable), process ceph-osd, pid 1048682
-46> 2019-08-11 17:11:01.970611 7ff4822f1d80 0 pidfile_write: ignore empty --pid-file
-45> 2019-08-11 17:11:01.991542 7ff4822f1d80 -1 WARNING: the following dangerous and experimental features are enabled: bluestore,rocksdb
-44> 2019-08-11 17:11:01.997597 7ff4822f1d80 0 load: jerasure load: lrc load: isa
-43> 2019-08-11 17:11:01.997710 7ff4822f1d80 1 bdev create path /var/lib/ceph/osd/ceph-4/block type kernel
-42> 2019-08-11 17:11:01.997723 7ff4822f1d80 1 bdev(0x564774656c00 /var/lib/ceph/osd/ceph-4/block) open path /var/lib/ceph/osd/ceph-4/block
-41> 2019-08-11 17:11:01.998127 7ff4822f1d80 1 bdev(0x564774656c00 /var/lib/ceph/osd/ceph-4/block) open size 858887553024 (0xc7f9b00000, 800GiB) block_size 4096 (4KiB) non-rotational
-40> 2019-08-11 17:11:01.998231 7ff4822f1d80 1 bdev(0x564774656c00 /var/lib/ceph/osd/ceph-4/block) close
-39> 2019-08-11 17:11:02.265144 7ff4822f1d80 1 bdev create path /var/lib/ceph/osd/ceph-4/block type kernel
-38> 2019-08-11 17:11:02.265177 7ff4822f1d80 1 bdev(0x564774658a00 /var/lib/ceph/osd/ceph-4/block) open path /var/lib/ceph/osd/ceph-4/block
-37> 2019-08-11 17:11:02.265695 7ff4822f1d80 1 bdev(0x564774658a00 /var/lib/ceph/osd/ceph-4/block) open size 858887553024 (0xc7f9b00000, 800GiB) block_size 4096 (4KiB) non-rotational
-36> 2019-08-11 17:11:02.266233 7ff4822f1d80 1 bdev create path /var/lib/ceph/osd/ceph-4/block.db type kernel
-35> 2019-08-11 17:11:02.266256 7ff4822f1d80 1 bdev(0x564774589a00 /var/lib/ceph/osd/ceph-4/block.db) open path /var/lib/ceph/osd/ceph-4/block.db
-34> 2019-08-11 17:11:02.266812 7ff4822f1d80 1 bdev(0x564774589a00 /var/lib/ceph/osd/ceph-4/block.db) open size 29999759360 (0x6fc200000, 27.9GiB) block_size 4096 (4KiB) non-rotational
-33> 2019-08-11 17:11:02.266998 7ff4822f1d80 1 bdev create path /var/lib/ceph/osd/ceph-4/block type kernel
-32> 2019-08-11 17:11:02.267015 7ff4822f1d80 1 bdev(0x564774659a00 /var/lib/ceph/osd/ceph-4/block) open path /var/lib/ceph/osd/ceph-4/block
-31> 2019-08-11 17:11:02.267412 7ff4822f1d80 1 bdev(0x564774659a00 /var/lib/ceph/osd/ceph-4/block) open size 858887553024 (0xc7f9b00000, 800GiB) block_size 4096 (4KiB) non-rotational
-30> 2019-08-11 17:11:02.298355 7ff4822f1d80 0 set rocksdb option compaction_readahead_size = 2MB
-29> 2019-08-11 17:11:02.298368 7ff4822f1d80 0 set rocksdb option compaction_style = kCompactionStyleLevel
-28> 2019-08-11 17:11:02.299628 7ff4822f1d80 0 set rocksdb option compaction_threads = 32
-27> 2019-08-11 17:11:02.299648 7ff4822f1d80 0 set rocksdb option compression = kNoCompression
-26> 2019-08-11 17:11:02.299993 7ff4822f1d80 0 set rocksdb option flusher_threads = 8
-25> 2019-08-11 17:11:02.300006 7ff4822f1d80 0 set rocksdb option level0_file_num_compaction_trigger = 64
-24> 2019-08-11 17:11:02.300011 7ff4822f1d80 0 set rocksdb option level0_slowdown_writes_trigger = 128
-23> 2019-08-11 17:11:02.300017 7ff4822f1d80 0 set rocksdb option level0_stop_writes_trigger = 256
-22> 2019-08-11 17:11:02.300022 7ff4822f1d80 0 set rocksdb option max_background_compactions = 64
-21> 2019-08-11 17:11:02.300027 7ff4822f1d80 0 set rocksdb option max_bytes_for_level_base = 2GB
-20> 2019-08-11 17:11:02.300034 7ff4822f1d80 0 set rocksdb option max_write_buffer_number = 64
-19> 2019-08-11 17:11:02.300039 7ff4822f1d80 0 set rocksdb option min_write_buffer_number_to_merge = 32
-18> 2019-08-11 17:11:02.300044 7ff4822f1d80 0 set rocksdb option recycle_log_file_num = 64
-17> 2019-08-11 17:11:02.300048 7ff4822f1d80 0 set rocksdb option target_file_size_base = 4MB
-16> 2019-08-11 17:11:02.300053 7ff4822f1d80 0 set rocksdb option write_buffer_size = 4MB
-15> 2019-08-11 17:11:02.300093 7ff4822f1d80 0 set rocksdb option compaction_readahead_size = 2MB
-14> 2019-08-11 17:11:02.300103 7ff4822f1d80 0 set rocksdb option compaction_style = kCompactionStyleLevel
-13> 2019-08-11 17:11:02.300110 7ff4822f1d80 0 set rocksdb option compaction_threads = 32
-12> 2019-08-11 17:11:02.300121 7ff4822f1d80 0 set rocksdb option compression = kNoCompression
-11> 2019-08-11 17:11:02.300129 7ff4822f1d80 0 set rocksdb option flusher_threads = 8
-10> 2019-08-11 17:11:02.300135 7ff4822f1d80 0 set rocksdb option level0_file_num_compaction_trigger = 64
-9> 2019-08-11 17:11:02.300142 7ff4822f1d80 0 set rocksdb option level0_slowdown_writes_trigger = 128
-8> 2019-08-11 17:11:02.300146 7ff4822f1d80 0 set rocksdb option level0_stop_writes_trigger = 256
-7> 2019-08-11 17:11:02.300150 7ff4822f1d80 0 set rocksdb option max_background_compactions = 64
-6> 2019-08-11 17:11:02.300155 7ff4822f1d80 0 set rocksdb option max_bytes_for_level_base = 2GB
-5> 2019-08-11 17:11:02.300159 7ff4822f1d80 0 set rocksdb option max_write_buffer_number = 64
-4> 2019-08-11 17:11:02.300166 7ff4822f1d80 0 set rocksdb option min_write_buffer_number_to_merge = 32
-3> 2019-08-11 17:11:02.300176 7ff4822f1d80 0 set rocksdb option recycle_log_file_num = 64
-2> 2019-08-11 17:11:02.300185 7ff4822f1d80 0 set rocksdb option target_file_size_base = 4MB
-1> 2019-08-11 17:11:02.300193 7ff4822f1d80 0 set rocksdb option write_buffer_size = 4MB
0> 2019-08-11 17:11:02.819067 7ff4822f1d80 -1 *** Caught signal (Aborted) **
in thread 7ff4822f1d80 thread_name:ceph-osd
No data to display
Actions