Ceph » Messengers

I can consistently reproduce this mds crash against master and nautilus vstart clusters, using the following test:
https://github.com/ddiss/ceph/commit/67938a7a8089226d9e32e642169c8ff4d5c3c9db

(gdb) bt
#0 0x000055f57cf25a87 in Server::handle_client_getattr (this=0x7f02bc002980, mdr=..., is_lookup=true) at /home/ddiss/isms/ceph/src/mds/Server.cc:3805
#1 0x000055f57cf19c3e in Server::dispatch_client_request (this=0x7f02bc002980, mdr=...) at /home/ddiss/isms/ceph/src/mds/Server.cc:2480
#2 0x000055f57cf18501 in Server::handle_client_request (this=0x7f02bc002980, req=...) at /home/ddiss/isms/ceph/src/mds/Server.cc:2353
#3 0x000055f57cf008d9 in Server::dispatch (this=0x7f02bc002980, m=...) at /home/ddiss/isms/ceph/src/mds/Server.cc:317
#4 0x000055f57ce8f906 in MDSRank::handle_deferrable_message (this=0x7f02ac032d48, m=...) at /home/ddiss/isms/ceph/src/mds/MDSRank.cc:1162
#5 0x000055f57ce8e3b9 in MDSRank::_dispatch (this=0x7f02ac032d48, m=..., new_msg=true) at /home/ddiss/isms/ceph/src/mds/MDSRank.cc:1021
#6 0x000055f57ce8df3f in MDSRankDispatcher::ms_dispatch (this=0x7f02ac032d40, m=...) at /home/ddiss/isms/ceph/src/mds/MDSRank.cc:1003
#7 0x000055f57ce6c074 in MDSDaemon::ms_dispatch2 (this=0x55f57e024e10, m=...) at /home/ddiss/isms/ceph/src/mds/MDSDaemon.cc:905
#8 0x00007f02d9f06da1 in Messenger::ms_deliver_dispatch (this=0x55f57e020570, m=...) at /home/ddiss/isms/ceph/src/msg/Messenger.h:693
#9 0x00007f02d9f059e6 in DispatchQueue::entry (this=0x55f57e020908) at /home/ddiss/isms/ceph/src/msg/DispatchQueue.cc:199
#10 0x00007f02da0627e8 in DispatchQueue::DispatchThread::entry (this=0x55f57e020a98) at /home/ddiss/isms/ceph/src/msg/DispatchQueue.h:101
#11 0x00007f02d9cd60a2 in Thread::entry_wrapper (this=0x55f57e020a98) at /home/ddiss/isms/ceph/src/common/Thread.cc:84
#12 0x00007f02d9cd6020 in Thread::_entry_func (arg=0x55f57e020a98) at /home/ddiss/isms/ceph/src/common/Thread.cc:71
#13 0x00007f02d7c0d569 in start_thread () from /lib64/libpthread.so.0
#14 0x00007f02d6c309ef in clone () from /lib64/libc.so.6
(gdb) l /home/ddiss/isms/ceph/src/mds/Server.cc:3801
3796 // value for them. (currently this matters for xattrs and inline data)
3797 mdr->getattr_caps = mask;
3798
3799 mds->balancer->hit_inode(ref, META_POP_IRD, req->get_source().num());
3800
3801 // reply
3802 dout(10) << "reply to stat on " << req << dendl;
3803 mdr->tracei = ref;
3804 if (is_lookup)
3805 mdr->tracedn = mdr->dn⁰.back();
(gdb) p mdr->dn
$1 = {{<std::_Vector_base<CDentry, std::allocator<CDentry*> >> = {
M_impl = {<std::allocator<CDentry*>> = {<_gnu_cxx::new_allocator<CDentry*>> = {<No data fields>}, <No data fields>}, M_start = 0x0, _M_finish = 0x0,
_M_end_of_storage = 0x0}}, <No data fields>}, {<std::_Vector_base<CDentry*, std::allocator<CDentry*> >> = {
_M_impl = {<std::allocator<CDentry*>> = {<_gnu_cxx::new_allocator<CDentry*>> = {<No data fields>}, <No data fields>}, _M_start = 0x0, _M_finish = 0x0,
_M_end_of_storage = 0x0}}, <No data fields>}}

(gdb) p mdr
$2 = (MDRequestImpl &) @0x7f02ac0ee530: {<MutationImpl> = {<TrackedOp> = {<boost::intrusive::list_base_hook<>> = {<boost::intrusive::generic_hook<(boost::intrusive::algo_types)0, boost::intrusive::list_node_traits<void>, boost::intrusive::dft_tag, (boost::intrusive::link_mode_type)1, (boost::intrusive::base_hook_type)1>> = {<boost::intrusive::node_holder<boost::intrusive::list_node<void*>, boost::intrusive::dft_tag, 1>> = {<boost::intrusive::list_node<void*>> = {next_ = 0x0,
prev_ = 0x0}, <No data fields>}, <boost::intrusive::hook_tags_definer<boost::intrusive::generic_hook<(boost::intrusive::algo_types)0, boost::intrusive::list_node_traits<void*>, boost::intrusive::dft_tag, (boost::intrusive::link_mode_type)1, (boost::intrusive::base_hook_type)1>, 1>> = {<No data fields>}, <No data fields>}, <No data fields>}, vptr.TrackedOp = 0x55f57d979440 <vtable for MDRequestImpl+16>,
tracker_item = {<boost::intrusive::generic_hook<(boost::intrusive::algo_types)0, boost::intrusive::list_node_traits<void*>, boost::intrusive::member_tag, (boost::intrusive::link_mode_type)1, (boost::intrusive::base_hook_type)0>> = {<boost::intrusive::list_node<void*>> = {next = 0x7f02bc002f28,
prev_ = 0x7f02bc002f28}, <boost::intrusive::hook_tags_definer<boost::intrusive::generic_hook<(boost::intrusive::algo_types)0, boost::intrusive::list_node_traits<void*>, boost::intrusive::member_tag, (boost::intrusive::link_mode_type)1, (boost::intrusive::base_hook_type)0>, 0>> = {<No data fields>}, <No data fields>}, <No data fields>}, tracker = 0x7f02ac033020, nref = {<std::__atomic_base<int>> = {static S_alignment = 4, _M_i = 2}, static is_always_lock_free = true}, initiated_at = {tv = {
tv_sec = 1578926979, tv_nsec = 250373651}}, events = {<std::_Vector_base<TrackedOp::Event, std::allocator<TrackedOp::Event> >> = {
_M_impl = {<std::allocator<TrackedOp::Event>> = {<_gnu_cxx::new_allocator<TrackedOp::Event>> = {<No data fields>}, <No data fields>}, M_start = 0x7f02ac0ba4a0,
_M_finish = 0x7f02ac0ba590, _M_end_of_storage = 0x7f02ac0ba7c0}}, <No data fields>}, lock = {<ceph::mutex_debug_detail::mutex_debugging_base> = {group = {
static npos = 18446744073709551615, _M_dataplus = {<std::allocator<char>> = {<_gnu_cxx::new_allocator<char>> = {<No data fields>}, <No data fields>},
M_p = 0x7f02ac0ee598 "TrackedOp::lock"}, _M_string_length = 15, {_M_local_buf = "TrackedOp::lock", _M_allocated_capacity = 5720808938570609236}}, id = 81,
lockdep = true, backtrace = false, nlock = 0, locked_by = {_M_thread = 0}}, m = {_data = {__lock = 0, _count = 0, __owner = 0, __nusers = 0, __kind = 2,
__spins = 0, __elision = 0, __list = {_prev = 0x0, _next = 0x0}}, __size = '\000' <repeats 16 times>, "\002", '\000' <repeats 22 times>, __align = 0},
static recursive = false}, seq = 5, warn_interval_multiplier = 1, state = {<std::_atomic_base<int>> = {static S_alignment = 4, _M_i = 1},
static is_always_lock_free = true}, desc_str = {static npos = 18446744073709551615,
_M_dataplus = {<std::allocator<char>> = {<_gnu_cxx::new_allocator<char>> = {<No data fields>}, <No data fields>}, M_p = 0x7f02ac0ee608 ""}, _M_string_length = 0, {
_M_local_buf = "\000\345\016\254\002\177\000\000\340\345\016\254\002\177\000", _M_allocated_capacity = 139649453319424}}, desc = 0x0, want_new_desc = {_M_base = {
static _S_alignment = 1, _M_i = false}, static is_always_lock_free = true}, osd_trace = {<No data fields>}, pg_trace = {<No data fields>},
store_trace = {<No data fields>}, journal_trace = {<No data fields>}}, reqid = {name = {_type = 8 '\b', _num = 4147, static TYPE_MON = 1, static TYPE_MDS = 2,
static TYPE_OSD = 4, static TYPE_CLIENT = 8, static TYPE_MGR = 16, static NEW = 1}, tid = 3}, attempt = 0, ls = 0x0, slave_to_mds = -1, object_states = {
_M_h = {<std::_detail::_Hashtable_base<MDSCacheObject*, std::pair<MDSCacheObject* const, MutationImpl::ObjectState>, std::__detail::_Select1st, std::equal_to<MDSCacheObject*>, std::hash<MDSCacheObject*>, std::__detail::_Mod_range_hashing, std::__detail::_Default_ranged_hash, std::__detail::_Hashtable_traits<false, false, true> >> = {<std::__detail::_Hash_code_base<MDSCacheObject*, std::pair<MDSCacheObject* const, MutationImpl::ObjectState>, std::__detail::_Select1st, std::hash<MDSCacheObject*>, std::__detail::_Mod_range_hashing, std::__detail::_Default_ranged_hash, false>> = {<std::__detail::_Hashtable_ebo_helper<0, std::__detail::_Select1st, true>> = {<std::__detail::_Select1st> = {<No data fields>}, <No data fields>}, <std::__detail::_Hashtable_ebo_helper<1, std::hash<MDSCacheObject*>, true>> = {<std::hash<MDSCacheObject*>> = {<std::__hash_base<unsigned long, MDSCacheObject*>> = {<No data fields>}, <No data fields>}, <No data fields>}, <std::__detail::_Hashtable_ebo_helper<2, std::__detail::_Mod_range_hashing, true>> = {<std::__detail::_Mod_range_hashing> = {<No data fields>}, <No data fields>}, <No data fields>}, <std::__detail::_Hashtable_ebo_helper<0, std::equal_to<MDSCacheObject*>, true>> = {<std::equal_to<MDSCacheObject*>> = {<std::binary_function<MDSCacheObject*, MDSCacheObject*, bool>> = {<No data fields>}, <No data fields>}, <No data fields>}, <No data fields>}, <std::__detail::_Map_base<MDSCacheObject*, std::pair<MDSCacheObject* const, MutationImpl::ObjectState>, std::allocator<std::pair<MDSCacheObject* const, MutationImpl::ObjectState> >, std::__detail::_Select1st, std::equal_to<MDSCacheObject*>, std::hash<MDSCacheObject*>, std::__detail::_Mod_range_hashing, std::__detail::_Default_ranged_hash, std::__detail::_Prime_rehash_policy, std::__detail::_Hashtable_traits<false, false, true>, true>> = {<No data fields>}, <std::__detail::_Insert<MDSCacheObject*, std::pair<MDSCacheObject* const, MutationImpl::ObjectState>, std::allocator<std::pair<MDSCacheObject* const, MutationImpl::ObjectState> >, std::__detail::_Select1st, std::equal_to<MDSCacheObject*>, std::hash<MDSCacheObject*>, std::__detail::_Mod_range_hashing, std::__detail::_Default_ranged_hash, std::__detail::_Prime_rehash_policy, std::__detail::_Hashtable_traits<false, false, true>, false>> = {<std::__detail::_Insert_base<MDSCacheObject*, std::pair<MDSCacheObject* const, MutationImpl::ObjectState>, std::allocator<std::pair<MDSCacheObject* const, MutationImpl::ObjectState> >, std::__detail::_Select1st, std::equal_to<MDSCacheObject*>, std::hash<MDSCacheObject*>, std::__detail::_Mod_range_hashing, std::__detail::_Default_ranged_hash, std::__detail::_Prime_rehash_policy, std::__detail::_Hashtable_traits<false, false, true> >> = {<No data fields>}, <No data fields>}, <std::__detail::_Rehash_base<MDSCacheObject*, std::pair<MDSCacheObject* const, MutationImpl::ObjectState>, std::allocator<std::pair<MDSCacheObject* const, MutationImpl::ObjectState> >, std::__detail::_Select1st, std::equal_to<MDSCacheObject*>, std::hash<MDSCacheObject*>, std::__detail::_Mod_range_hashing, std::__detail::_Default_ranged_hash, std::__detail::_Prime_rehash_policy, std::__detail::_Hashtable_traits<false, false, true>, std::integral_constant<bool, true> >> = {<No data fields>}, <std::__detail::_Equality<MDSCacheObject*, std::pair<MDSCacheObject* const, MutationImpl::ObjectState>, std::allocator<std::pair<MDSCacheObject* const, MutationImpl::ObjectState> >, std::__detail::_Selec--Type <RET> for more, q to quit, c to continue without paging-
t1st, std::equal_to<MDSCacheObject*>, std::hash<MDSCacheObject*>, std::__detail::_Mod_range_hashing, std::__detail::_Default_ranged_hash, std::__detail::_Prime_rehash_policy, std::__detail::_Hashtable_traits<false, false, true>, true>> = {<No data fields>}, <std::__detail::_Hashtable_alloc<std::allocator<std::__detail::_Hash_node<std::pair<MDSCacheObject* const, MutationImpl::ObjectState>, false> > >> = {<std::__detail::_Hashtable_ebo_helper<0, std::allocator<std::__detail::_Hash_node<std::pair<MDSCacheObject* const, MutationImpl::ObjectState>, false> >, true>> = {<std::allocator<std::__detail::_Hash_node<std::pair<MDSCacheObject* const, MutationImpl::ObjectState>, false> >> = {<__gnu_cxx::new_allocator<std::__detail::_Hash_node<std::pair<MDSCacheObject* const, MutationImpl::ObjectState>, false> >> = {<No data fields>}, <No data fields>}, <No data fields>}, <No data fields>}, M_buckets = 0x7f02ac0ec150, _M_bucket_count = 3, _M_before_begin = {_M_nxt = 0x7f02ac0a3640}, _M_element_count = 1, _M_rehash_policy = {
static _S_growth_factor = 2, _M_max_load_factor = 1, _M_next_resize = 3}, _M_single_bucket = 0x0}}, num_pins = 1, num_auth_pins = 1, num_remote_auth_pins = 0,
stickydiri = 0x0, locks = {_M_t = {
_M_impl = {<std::allocator<std::_Rb_tree_node<MutationImpl::LockOp> >> = {<_gnu_cxx::new_allocator<std::_Rb_tree_node<MutationImpl::LockOp> >> = {<No data fields>}, <No data fields>}, <std::_Rb_tree_key_compare<std::less<MutationImpl::LockOp> >> = {
M_key_compare = {<std::binary_function<MutationImpl::LockOp, MutationImpl::LockOp, bool>> = {<No data fields>}, <No data fields>}}, <std::_Rb_tree_header> = {
_M_header = {_M_color = std::_S_red, _M_parent = 0x0, _M_left = 0x7f02ac0ee6b0, _M_right = 0x7f02ac0ee6b0}, _M_node_count = 0}, <No data fields>}}},
lock_cache = 0x0, lock_cache_disabled = false, last_locked = 0x0, locking = 0x0, locking_target_mds = -1, locking_state = 0, committing = false, aborted = false,
killed = false, projected_inodes = {<std::_cxx11::_List_base<CInode*, std::allocator<CInode*> >> = {
M_impl = {<std::allocator<std::_List_node<CInode*> >> = {<_gnu_cxx::new_allocator<std::_List_node<CInode*> >> = {<No data fields>}, <No data fields>},
M_node = {<std::_detail::_List_node_base> = {_M_next = 0x7f02ac0ee708, M_prev = 0x7f02ac0ee708}, _M_storage = {
_M_storage = "\000\000\000\000\000\000\000"}}}}, <No data fields>}, projected_fnodes = {<std::_Vector_base<CDir*, std::allocator<CDir*> >> = {
_M_impl = {<std::allocator<CDir*>> = {<_gnu_cxx::new_allocator<CDir*>> = {<No data fields>}, <No data fields>}, M_start = 0x0, _M_finish = 0x0,
_M_end_of_storage = 0x0}}, <No data fields>}, updated_locks = {<std::_cxx11::_List_base<ScatterLock*, std::allocator<ScatterLock*> >> = {
M_impl = {<std::allocator<std::_List_node<ScatterLock*> >> = {<_gnu_cxx::new_allocator<std::_List_node<ScatterLock*> >> = {<No data fields>}, <No data fields>},
M_node = {<std::_detail::_List_node_base> = {_M_next = 0x7f02ac0ee738, M_prev = 0x7f02ac0ee738}, _M_storage = {
_M_storage = "\000\000\000\000\000\000\000"}}}}, <No data fields>}, dirty_cow_inodes = {<std::_cxx11::_List_base<CInode*, std::allocator<CInode*> >> = {
M_impl = {<std::allocator<std::_List_node<CInode*> >> = {<_gnu_cxx::new_allocator<std::_List_node<CInode*> >> = {<No data fields>}, <No data fields>},
M_node = {<std::_detail::_List_node_base> = {_M_next = 0x7f02ac0ee750, M_prev = 0x7f02ac0ee750}, _M_storage = {
_M_storage = "\000\000\000\000\000\000\000"}}}}, <No data fields>},
dirty_cow_dentries = {<std::_cxx11::_List_base<std::pair<CDentry*, unsigned long>, std::allocator<std::pair<CDentry*, unsigned long> > >> = {
M_impl = {<std::allocator<std::_List_node<std::pair<CDentry*, unsigned long> > >> = {<_gnu_cxx::new_allocator<std::_List_node<std::pair<CDentry*, unsigned long> > >> = {<No data fields>}, <No data fields>}, M_node = {<std::_detail::_List_node_base> = {_M_next = 0x7f02ac0ee768, M_prev = 0x7f02ac0ee768}, _M_storage = {
_M_storage = "\000\000\000\000\000\000\000"}}}}, <No data fields>}, mds_stamp = {tv = {tv_sec = 1578926979, tv_nsec = 250481706}}, op_stamp = {tv = {
tv_sec = 1578926979, tv_nsec = 250242050}}}, _more = 0x0, session = 0x7f02ac0ed7b0, item_session_request = {_prev = 0x7f02ac0ed9d8, _next = 0x7f02ac0ed9d8},
client_request = {px = 0x7f02c0004b10}, dir_root = {{val = 0}, {val = 0}}, dir_depth = {-1, -1}, dir_layout = {stripe_unit = 0, stripe_count = 0, object_size = 0,
pool_id = -1, pool_ns = {static npos = 18446744073709551615,
_M_dataplus = {<std::allocator<char>> = {<_gnu_cxx::new_allocator<char>> = {<No data fields>}, <No data fields>}, _M_p = 0x7f02ac0ee7f8 ""}, _M_string_length = 0, {
_M_local_buf = "\000\347\016\254\002\177\000\000\000\347\016\254\002\177\000", _M_allocated_capacity = 139649453319936}}}, dn = { {<std::_Vector_base<CDentry*, std::allocator<CDentry*> >> = {
M_impl = {<std::allocator<CDentry*>> = {<_gnu_cxx::new_allocator<CDentry*>> = {<No data fields>}, <No data fields>}, M_start = 0x0, _M_finish = 0x0,
_M_end_of_storage = 0x0}}, <No data fields>}, {<std::_Vector_base<CDentry*, std::allocator<CDentry*> >> = {
_M_impl = {<std::allocator<CDentry*>> = {<_gnu_cxx::new_allocator<CDentry*>> = {<No data fields>}, <No data fields>}, M_start = 0x0, _M_finish = 0x0,
_M_end_of_storage = 0x0}}, <No data fields>}}, in = {0x7f02ac0609a0, 0x0}, straydn = 0x0, snapid = {val = 18446744073709551615}, tracei = 0x7f02ac0609a0,
tracedn = 0x0, alloc_ino = {val = 0}, used_prealloc_ino = {val = 0}, prealloc_inos = {_size = 0, m = {_M_t = {
_M_impl = {<std::allocator<std::_Rb_tree_node<std::pair<inodeno_t const, inodeno_t> > >> = {<_gnu_cxx::new_allocator<std::_Rb_tree_node<std::pair<inodeno_t const, inodeno_t> > >> = {<No data fields>}, <No data fields>}, <std::_Rb_tree_key_compare<std::less<inodeno_t> >> = {
M_key_compare = {<std::binary_function<inodeno_t, inodeno_t, bool>> = {<No data fields>}, <No data fields>}}, <std::_Rb_tree_header> = {_M_header = {
_M_color = std::_S_red, _M_parent = 0x0, _M_left = 0x7f02ac0ee888, _M_right = 0x7f02ac0ee888}, _M_node_count = 0}, <No data fields>}}}}, snap_caps = 0,
getattr_caps = 1, no_early_reply = false, did_early_reply = false, o_trunc = false, has_completed = false, reply_extra_bl = {_buffers = {_root = {next = 0x7f02ac0ee8c0},
--Type <RET> for more, q to quit, c to continue without paging--
_tail = 0x7f02ac0ee8c0, _size = 0}, _carriage = 0x7f02e2fab680 <ceph::buffer::v14_2_0::list::always_empty_bptr>, _len = 0, _memcopy_count = 0,
last_p = {<ceph::buffer::v14_2_0::list::iterator_impl<false>> = {bl = 0x7f02ac0ee8c0, ls = 0x7f02ac0ee8c0, p = {cur = 0x7f02ac0ee8c0}, off = 0,
p_off = 0}, <No data fields>}, static always_empty_bptr = {_raw = 0x0, _off = 0, _len = 0}, static CLAIM_DEFAULT = 0, static CLAIM_ALLOW_NONSHAREABLE = 1},
cap_releases = {_M_t = {
_M_impl = {<std::allocator<std::_Rb_tree_node<std::pair<vinodeno_t const, unsigned int> > >> = {<_gnu_cxx::new_allocator<std::_Rb_tree_node<std::pair<vinodeno_t const, unsigned int> > >> = {<No data fields>}, <No data fields>}, <std::_Rb_tree_key_compare<std::less<vinodeno_t> >> = {
M_key_compare = {<std::binary_function<vinodeno_t, vinodeno_t, bool>> = {<No data fields>}, <No data fields>}}, <std::_Rb_tree_header> = {_M_header = {
_M_color = std::_S_red, _M_parent = 0x0, _M_left = 0x7f02ac0ee910, _M_right = 0x7f02ac0ee910}, _M_node_count = 0}, <No data fields>}}}, slave_request = {
px = 0x0}, internal_op = -1, internal_op_finish = 0x0, internal_op_private = 0x0, retry = 0, is_batch_head = false, waited_for_osdmap = false,
batch_reqs = {<std::_Vector_base<boost::intrusive_ptr<MDRequestImpl>, std::allocator<boost::intrusive_ptr<MDRequestImpl> > >> = {
_M_impl = {<std::allocator<boost::intrusive_ptr<MDRequestImpl> >> = {<_gnu_cxx::new_allocator<boost::intrusive_ptr<MDRequestImpl> >> = {<No data fields>}, <No data fields>}, M_start = 0x0, _M_finish = 0x0, _M_end_of_storage = 0x0}}, <No data fields>}, msg_lock = {af = {<std::_atomic_flag_base> = {_M_i = false}, <No data fields>}}}