Ceph » bluestore

i'm guessing this is the same heap corruption we've been seeing, but logging it anyway

(gdb) bt
#0  0x00007f60dfb8c74d in tc_newarray () from /usr/lib/x86_64-linux-gnu/libtcmalloc.so.4
#1  0x000055c9fd5fe2f2 in std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> >::_M_construct<char*> (this=this@entry=0x7f60b4d1e280, __beg=0x55ca244a9b00 "ceph version 14.0.1-3425-gff23e1e (ff23e1ed2868ffdfbd9f201f6ecbb8426b3aa0d7) nautilus (dev)������\223", __end=<optimized out>)
    at /usr/include/c++/7/bits/basic_string.tcc:219
#2  0x000055c9fd5fe762 in std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> >::_M_construct_aux<char*> (__end=<optimized out>, __beg=<optimized out>, this=0x7f60b4d1e280) at /usr/include/c++/7/bits/basic_string.h:236
#3  std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> >::_M_construct<char*> (__end=<optimized out>, __beg=<optimized out>, this=0x7f60b4d1e280) at /usr/include/c++/7/bits/basic_string.h:255
#4  std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> >::basic_string<char*, void> (__a=..., __end=<optimized out>, __beg=<optimized out>, this=0x7f60b4d1e280) at /usr/include/c++/7/bits/basic_string.h:607
#5  std::__cxx11::basic_stringbuf<char, std::char_traits<char>, std::allocator<char> >::str (this=<optimized out>, this=<optimized out>) at /usr/include/c++/7/sstream:173
#6  std::__cxx11::basic_ostringstream<char, std::char_traits<char>, std::allocator<char> >::str (this=0x7f60b4d1e2a0) at /usr/include/c++/7/sstream:638
#7  pretty_version_to_str[abi:cxx11]() () at ./src/common/version.cc:48
#8  0x000055c9fd4784aa in ceph::BackTrace::print (this=this@entry=0x7f60b4d1ec00, out=...) at ./src/common/BackTrace.cc:19
#9  0x000055c9fd45e286 in handle_fatal_signal (signum=11) at ./src/global/signal_handler.cc:169
#10 <signal handler called>
#11 0x00007f60dfb8c74d in tc_newarray () from /usr/lib/x86_64-linux-gnu/libtcmalloc.so.4
#12 0x000055c9fd2b9d0b in __gnu_cxx::new_allocator<std::_Rb_tree_node<std::pair<std::__cxx11::basic_string<char, std::char_traits<char>, mempool::pool_allocator<(mempool::pool_index_t)4, char> > const, ceph::buffer::ptr> > >::allocate (this=<optimized out>, __n=1) at /usr/include/c++/7/ext/new_allocator.h:111
#13 std::allocator_traits<std::allocator<std::_Rb_tree_node<std::pair<std::__cxx11::basic_string<char, std::char_traits<char>, mempool::pool_allocator<(mempool::pool_index_t)4, char> > const, ceph::buffer::ptr> > > >::allocate (__a=..., __n=1) at /usr/include/c++/7/bits/alloc_traits.h:436
#14 std::_Rb_tree<std::__cxx11::basic_string<char, std::char_traits<char>, mempool::pool_allocator<(mempool::pool_index_t)4, char> >, std::pair<std::__cxx11::basic_string<char, std::char_traits<char>, mempool::pool_allocator<(mempool::pool_index_t)4, char> > const, ceph::buffer::ptr>, std::_Select1st<std::pair<std::__cxx11::basic_string<char, std::char_traits<char>, mempool::pool_allocator<(mempool::pool_index_t)4, char> > const, ceph::buffer::ptr> >, std::less<std::__cxx11::basic_string<char, std::char_traits<char>, mempool::pool_allocator<(mempool::pool_index_t)4, char> > >, std::allocator<std::pair<std::__cxx11::basic_string<char, std::char_traits<char>, mempool::pool_allocator<(mempool::pool_index_t)4, char> > const, ceph::buffer::ptr> > >::_M_get_node (this=0x55ca0ee91f30, this@entry=0x55ca0ee92020) at /usr/include/c++/7/bits/stl_tree.h:588
#15 std::_Rb_tree<std::__cxx11::basic_string<char, std::char_traits<char>, mempool::pool_allocator<(mempool::pool_index_t)4, char> >, std::pair<std::__cxx11::basic_string<char, std::char_traits<char>, mempool::pool_allocator<(mempool::pool_index_t)4, char> > const, ceph::buffer::ptr>, std::_Select1st<std::pair<std::__cxx11::basic_string<char, std::char_traits<char>, mempool::pool_allocator<(mempool::pool_index_t)4, char> > const, ceph::buffer::ptr> >, std::less<std::__cxx11::basic_string<char, std::char_traits<char>, mempool::pool_allocator<(mempool::pool_index_t)4, char> > >, std::allocator<std::pair<std::__cxx11::basic_string<char, std::char_traits<char>, mempool::pool_allocator<(mempool::pool_index_t)4, char> > const, ceph::buffer::ptr> > >::_M_create_node<std::piecewise_construct_t const&, std::tuple<std::__cxx11::basic_string<char, std::char_traits<char>, mempool::pool_allocator<(mempool::pool_index_t)4, char> >&&>, std::tuple<> >(std::piecewise_construct_t const&, std::tuple<std::__cxx11::basic_string<char, std::char_traits<char>, mempool::pool_allocator<(mempool::pool_index_t)4, char> >&&>&&, std::tuple<>&&) (this=0x55ca0ee91f30, this@entry=0x55ca0ee92020) at /usr/include/c++/7/bits/stl_tree.h:642
#16 std::_Rb_tree<std::__cxx11::basic_string<char, std::char_traits<char>, mempool::pool_allocator<(mempool::pool_index_t)4, char> >, std::pair<std::__cxx11::basic_string<char, std::char_traits<char>, mempool::pool_allocator<(mempool::pool_index_t)4, char> > const, ceph::buffer::ptr>, std::_Select1st<std::pair<std::__cxx11::basic_string<char, std::char_traits<char>, mempool::pool_allocator<(mempool::pool_index_t)4, char> > const, ceph::buffer::ptr> >, std::less<std::__cxx11::basic_string<char, std::char_traits<char>, mempool::pool_allocator<(mempool::pool_index_t)4, char> > >, std::allocator<std::pair<std::__cxx11::basic_string<char, std::char_traits<char>, mempool::pool_allocator<(mempool::pool_index_t)4, char> > const, ceph::buffer::ptr> > >::_M_emplace_hint_unique<std::piecewise_construct_t const&, std::tuple<std::__cxx11::basic_string<char, std::char_traits<char>, mempool::pool_allocator<(mempool::pool_index_t)4, char> >&&>, std::tuple<> >(std::_Rb_tree_const_iterator<std::pair<std::__cxx11::basic_string<char, std::char_traits<char>, mempool::pool_allocator<(mempool::pool_index_t)4, char> > const, ceph::buffer::ptr> >, std::piecewise_construct_t const&, std::tuple<std::__cxx11::basic_string<char, std::char_traits<char>, mempool::pool_allocator<(mempool::pool_index_t)4, char> >&&>&&, std::tuple<>&&) (this=this@entry=0x55ca0ee91f30, __pos=__pos@entry={first = "", second = {_raw = 0x303a312c30303001, _off = 250158656, _len = 21962}}) at /usr/include/c++/7/bits/stl_tree.h:2398
#17 0x000055c9fd30be9b in std::map<std::__cxx11::basic_string<char, std::char_traits<char>, mempool::pool_allocator<(mempool::pool_index_t)4, char> >, ceph::buffer::ptr, std::less<std::__cxx11::basic_string<char, std::char_traits<char>, mempool::pool_allocator<(mempool::pool_index_t)4, char> > >, std::allocator<std::pair<std::__cxx11::basic_string<char, std::char_traits<char>, mempool::pool_allocator<(mempool::pool_index_t)4, char> > const, ceph::buffer::ptr> > >::operator[] (__k=..., this=0x55ca0ee91f30) at /usr/include/c++/7/bits/stl_map.h:512
#18 BlueStore::_setattrs (this=this@entry=0x55ca096e9000, txc=txc@entry=0x55ca0d8c8400, c=..., o=..., aset=std::map with 2 elements = {...}) at ./src/os/bluestore/BlueStore.cc:12677
#19 0x000055c9fd32b736 in BlueStore::_txc_add_transaction (this=this@entry=0x55ca096e9000, txc=txc@entry=0x55ca0d8c8400, t=t@entry=0x55ca18fa26e0) at ./src/os/bluestore/BlueStore.cc:11023
#20 0x000055c9fd3306b7 in BlueStore::queue_transactions (this=0x55ca096e9000, ch=..., tls=std::vector of length 1, capacity 1 = {...}, op=..., handle=0x0) at ./src/os/bluestore/BlueStore.cc:10753
#21 0x000055c9fd07fc44 in non-virtual thunk to PrimaryLogPG::queue_transactions(std::vector<ObjectStore::Transaction, std::allocator<ObjectStore::Transaction> >&, boost::intrusive_ptr<OpRequest>) ()
#22 0x000055c9fd17bc70 in ReplicatedBackend::submit_transaction (this=0x55ca0bdde780, soid=..., delta_stats=..., at_version=..., _t=..., trim_to=..., roll_forward_to=..., _log_entries=std::vector of length 1, capacity 1 = {...}, hset_history=..., on_all_commit=0x55ca0fa200e0, tid=64783, reqid=..., orig_op=...)
    at ./src/osd/ReplicatedBackend.cc:502
#23 0x000055c9fcfd86d6 in PrimaryLogPG::issue_repop (this=this@entry=0x55ca0be0c000, repop=repop@entry=0x55ca0c8fe580, ctx=ctx@entry=0x55ca11123600) at ./src/osd/PrimaryLogPG.cc:10578
#24 0x000055c9fd03c76a in PrimaryLogPG::execute_ctx (this=this@entry=0x55ca0be0c000, ctx=ctx@entry=0x55ca11123600) at ./src/osd/PrimaryLogPG.cc:4111
#25 0x000055c9fd04076e in PrimaryLogPG::do_op (this=this@entry=0x55ca0be0c000, op=...) at ./src/osd/PrimaryLogPG.cc:2431
#26 0x000055c9fd042758 in PrimaryLogPG::do_request (this=0x55ca0be0c000, op=..., handle=...) at ./src/osd/PrimaryLogPG.cc:1868
#27 0x000055c9fce6dd93 in OSD::dequeue_op (this=this@entry=0x55ca097b4000, pg=..., op=..., handle=...) at ./src/osd/OSD.cc:9687
#28 0x000055c9fd117962 in PGOpItem::run (this=<optimized out>, osd=0x55ca097b4000, sdata=<optimized out>, pg=..., handle=...) at ./src/osd/OpQueueItem.cc:24
#29 0x000055c9fce8baf5 in OpQueueItem::run (handle=..., pg=..., sdata=<optimized out>, osd=<optimized out>, this=0x7f60b4d264e0) at ./src/osd/OpQueueItem.h:134
#30 OSD::ShardedOpWQ::_process (this=0x55ca097b51f8, thread_index=<optimized out>, hb=<optimized out>) at ./src/osd/OSD.cc:10862
#31 0x000055c9fd4b613c in ShardedThreadPool::shardedthreadpool_worker (this=0x55ca097b4ad0, thread_index=14) at ./src/common/WorkQueue.cc:311
#32 0x000055c9fd4b9210 in ShardedThreadPool::WorkThreadSharded::entry (this=<optimized out>) at ./src/common/WorkQueue.h:699
#33 0x00007f60ddc786db in start_thread (arg=0x7f60b4d29700) at pthread_create.c:463
#34 0x00007f60dca1388f in clone () at ../sysdeps/unix/sysv/linux/x86_64/clone.S:95
(gdb) f 23
#23 0x000055c9fcfd86d6 in PrimaryLogPG::issue_repop (this=this@entry=0x55ca0be0c000, repop=repop@entry=0x55ca0c8fe580, ctx=ctx@entry=0x55ca11123600) at ./src/osd/PrimaryLogPG.cc:10578
10578   ./src/osd/PrimaryLogPG.cc: No such file or directory.
(gdb) p pgid
p ingoNo symbol "pgid" in current context.
(gdb) p info.pgid
$1 = {pgid = {m_pool = 4, m_seed = 6, static calc_name_buf_size = 36 '$'}, shard = {id = -1 '�', static NO_SHARD = {id = -1 '�', static NO_SHARD = <same as static member of an already seen type>}}, static calc_name_buf_size = 40 '('}