Bug #24974
Segmentation fault in tcmalloc::ThreadCache::ReleaseToCentralCache()
Status:
New
Priority:
Normal
Assignee:
-
Category:
-
Target version:
-
% Done:
0%
Source:
Tags:
Backport:
Regression:
No
Severity:
3 - minor
Reviewed:
Affected Versions:
ceph-qa-suite:
Component(RADOS):
Pull request ID:
Crash signature (v1):
Crash signature (v2):
Description
#0 0x00007fe1d6e0a0b3 in tcmalloc::ThreadCache::ReleaseToCentralCache(tcmalloc::ThreadCache::FreeList*, unsigned long, int) () from /usr/lib/libtcmalloc.so.4 [Current thread is 1 (Thread 0x7fe1d1687700 (LWP 13962))] (gdb) bt #0 0x00007fe1d6e0a0b3 in tcmalloc::ThreadCache::ReleaseToCentralCache(tcmalloc::ThreadCache::FreeList*, unsigned long, int) () from /usr/lib/libtcmalloc.so.4 #1 0x00007fe1d6e0a16b in tcmalloc::ThreadCache::ListTooLong(tcmalloc::ThreadCache::FreeList*, unsigned long) () from /usr/lib/libtcmalloc.so.4 #2 0x00007fe1d6e1c818 in tc_deletearray () from /usr/lib/libtcmalloc.so.4 #3 0x00007fe1d8231889 in __cxa_demangle () from /usr/lib/ceph/libceph-common.so.0 #4 0x00007fe1d7dad500 in ceph::BackTrace::print(std::ostream&) const () from /usr/lib/ceph/libceph-common.so.0 #5 0x00005589602b888a in handle_fatal_signal (signum=11) at /build/ceph-13.2.0-301-gbfc94b5/src/global/signal_handler.cc:119 #6 <signal handler called> #7 0x00007fe1d6e0a0b3 in tcmalloc::ThreadCache::ReleaseToCentralCache(tcmalloc::ThreadCache::FreeList*, unsigned long, int) () from /usr/lib/libtcmalloc.so.4 #8 0x00007fe1d6e0a16b in tcmalloc::ThreadCache::ListTooLong(tcmalloc::ThreadCache::FreeList*, unsigned long) () from /usr/lib/libtcmalloc.so.4 #9 0x00007fe1d6e1c818 in tc_deletearray () from /usr/lib/libtcmalloc.so.4 #10 0x00007fe1d7d95a07 in std::__cxx11::_List_base<ceph::buffer::ptr, std::allocator<ceph::buffer::ptr> >::_M_clear() () from /usr/lib/ceph/libceph-common.so.0 #11 0x00007fe1d7f1f48b in PosixConnectedSocketImpl::send(ceph::buffer::list&, bool) () from /usr/lib/ceph/libceph-common.so.0 #12 0x00007fe1d7ef84b6 in AsyncConnection::_try_send(bool) () from /usr/lib/ceph/libceph-common.so.0 #13 0x00007fe1d7ef9b3e in AsyncConnection::write_message(Message*, ceph::buffer::list&, bool) () from /usr/lib/ceph/libceph-common.so.0 #14 0x00007fe1d7eff389 in AsyncConnection::handle_write() () from /usr/lib/ceph/libceph-common.so.0 #15 0x00007fe1d7f1a077 in EventCenter::process_events(unsigned int, std::chrono::duration<unsigned long, std::ratio<1l, 1000000000l> >*) () from /usr/lib/ceph/libceph-common.so.0 #16 0x00007fe1d7f1c9dc in ?? () from /usr/lib/ceph/libceph-common.so.0 #17 0x00007fe1d825258f in ?? () from /usr/lib/ceph/libceph-common.so.0 #18 0x00007fe1d64696ba in start_thread (arg=0x7fe1d1687700) at pthread_create.c:333 #19 0x00007fe1d5a7841d in clone () at ../sysdeps/unix/sysv/linux/x86_64/clone.S:109
/a/yuriw-2018-07-16_17:20:11-rados-wip-yuri4-testing-2018-07-16-1528-mimic-distro-basic-smithi/2788760/
History
#1 Updated by David Zafman almost 5 years ago
dzafman-2019-05-09_20:06:24-rados-wip-zafman-testing-distro-basic-smithi/3943901
Core was generated by `ceph-osd -f --cluster ceph -i 6'. Program terminated with signal SIGSEGV, Segmentation fault. #0 0x00007f00e0a50bcb in tcmalloc::ThreadCache::ReleaseToCentralCache(tcmalloc::ThreadCache::FreeList*, unsigned long, int) () from /usr/lib/x86_64-linux-gnu/libtcmalloc.so.4 [Current thread is 1 (Thread 0x7f00cac27700 (LWP 11964))] (gdb) bt #0 0x00007f00e0a50bcb in tcmalloc::ThreadCache::ReleaseToCentralCache(tcmalloc::ThreadCache::FreeList*, unsigned long, int) () from /usr/lib/x86_64-linux-gnu/libtcmalloc.so.4 #1 0x00007f00e0a50c9b in tcmalloc::ThreadCache::ListTooLong(tcmalloc::ThreadCache::FreeList*, unsigned long) () from /usr/lib/x86_64-linux-gnu/libtcmalloc.so.4 #2 0x00007f00e0a5f6f5 in tc_free () from /usr/lib/x86_64-linux-gnu/libtcmalloc.so.4 #3 0x000055b4c8bbf317 in __gnu_cxx::new_allocator<char>::deallocate (this=0x55b4d4fe6e88, __p=<optimized out>) at /usr/include/c++/8/ext/new_allocator.h:116 #4 std::allocator_traits<std::allocator<char> >::deallocate (__a=..., __n=<optimized out>, __p=<optimized out>) at /usr/include/c++/8/bits/alloc_traits.h:462 #5 std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> >::_M_destroy (__size=<optimized out>, this=0x55b4d4fe6e88) at /usr/include/c++/8/bits/basic_string.h:226 #6 std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> >::_M_dispose (this=0x55b4d4fe6e88) at /usr/include/c++/8/bits/basic_string.h:221 #7 std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> >::~basic_string (this=0x55b4d4fe6e88, __in_chrg=<optimized out>) at /usr/include/c++/8/bits/basic_string.h:647 #8 pidfh::~pidfh (this=0x55b4d4fe6e80, __in_chrg=<optimized out>) at ./src/global/pidfile.cc:52 #9 pidfile_remove () at ./src/global/pidfile.cc:210 #10 0x000055b4c8bc17c1 in handle_fatal_signal(int) () at ./src/global/signal_handler.cc:162 #11 <signal handler called> #12 0x00007f00e0a50bcb in tcmalloc::ThreadCache::ReleaseToCentralCache(tcmalloc::ThreadCache::FreeList*, unsigned long, int) () from /usr/lib/x86_64-linux-gnu/libtcmalloc.so.4 #13 0x00007f00e0a50c9b in tcmalloc::ThreadCache::ListTooLong(tcmalloc::ThreadCache::FreeList*, unsigned long) () from /usr/lib/x86_64-linux-gnu/libtcmalloc.so.4 #14 0x00007f00e0a5f6f5 in tc_free () from /usr/lib/x86_64-linux-gnu/libtcmalloc.so.4 #15 0x000055b4c864cd65 in ceph::buffer::v14_2_0::ptr_node::disposer::operator() (this=<optimized out>, delete_this=0x55b4d78d34e0) at ./src/include/buffer.h:401 #16 ceph::buffer::v14_2_0::list::buffers_t::clear_and_dispose (this=0x55b4db3e8558) at ./src/include/buffer.h:641 #17 ceph::buffer::v14_2_0::list::~list (this=0x55b4db3e8558, __in_chrg=<optimized out>) at ./src/include/buffer.h:990 #18 std::pair<unsigned int const, ceph::buffer::v14_2_0::list>::~pair (this=0x55b4db3e8550, __in_chrg=<optimized out>) at /usr/include/c++/8/bits/stl_pair.h:198 #19 __gnu_cxx::new_allocator<std::_Rb_tree_node<std::pair<unsigned int const, ceph::buffer::v14_2_0::list> > >::destroy<std::pair<unsigned int const, ceph::buffer::v14_2_0::list> > (this=0x55b4d7a52778, __p=0x55b4db3e8550) at /usr/include/c++/8/ext/new_allocator.h:140 #20 std::allocator_traits<std::allocator<std::_Rb_tree_node<std::pair<unsigned int const, ceph::buffer::v14_2_0::list> > > >::destroy<std::pair<unsigned int const, ceph::buffer::v14_2_0::list> > (__a=..., __p=0x55b4db3e8550) at /usr/include/c++/8/bits/alloc_traits.h:487 #21 std::_Rb_tree<unsigned int, std::pair<unsigned int const, ceph::buffer::v14_2_0::list>, std::_Select1st<std::pair<unsigned int const, ceph::buffer::v14_2_0::list> >, std::less<unsigned int>, std::allocator<std::pair<unsigned int const, ceph::buffer::v14_2_0::list> > >::_M_destroy_node (this=0x55b4d7a52778, __p=0x55b4db3e8530) at /usr/include/c++/8/bits/stl_tree.h:661 #22 std::_Rb_tree<unsigned int, std::pair<unsigned int const, ceph::buffer::v14_2_0::list>, std::_Select1st<std::pair<unsigned int const, ceph::buffer::v14_2_0::list> >, std::less<unsigned int>, std::allocator<std::pair<unsigned int const, ceph::buffer::v14_2_0::list> > >::_M_drop_node (this=0x55b4d7a52778, __p=0x55b4db3e8530) at /usr/include/c++/8/bits/stl_tree.h:669 #23 std::_Rb_tree<unsigned int, std::pair<unsigned int const, ceph::buffer::v14_2_0::list>, std::_Select1st<std::pair<unsigned int const, ceph::buffer::v14_2_0::list> >, std::less<unsigned int>, std::allocator<std::pair<unsigned int const, ceph::buffer::v14_2_0::list> > >::_M_erase (this=this@entry=0x55b4d7a52778, __x=0x55b4db3e8530) at /usr/include/c++/8/bits/stl_tree.h:1874 #24 0x000055b4c864cd2d in std::_Rb_tree<unsigned int, std::pair<unsigned int const, ceph::buffer::v14_2_0::list>, std::_Select1st<std::pair<unsigned int const, ceph::buffer::v14_2_0::list> >, std::less<unsigned int>, std::allocator<std::pair<unsigned int const, ceph::buffer::v14_2_0::list> > >::_M_erase (this=this@entry=0x55b4d7a52778, __x=0x55b4db3e6e70) at /usr/include/c++/8/bits/stl_tree.h:1866 #25 0x000055b4c867dc69 in std::_Rb_tree<unsigned int, std::pair<unsigned int const, ceph::buffer::v14_2_0::list>, std::_Select1st<std::pair<unsigned int const, ceph::buffer::v14_2_0::list> >, std::less<unsigned int>, std::allocator<std::pair<unsigned int const, ceph::buffer::v14_2_0::list> > >::~_Rb_tree (this=0x55b4d7a52778, __in_chrg=<optimized out>) at /usr/include/c++/8/bits/stl_tree.h:964 #26 std::map<unsigned int, ceph::buffer::v14_2_0::list, std::less<unsigned int>, std::allocator<std::pair<unsigned int const, ceph::buffer::v14_2_0::list> > >::~map ( this=0x55b4d7a52778, __in_chrg=<optimized out>) at /usr/include/c++/8/bits/stl_map.h:300 #27 MOSDMap::~MOSDMap (this=0x55b4d7a52580, __in_chrg=<optimized out>) at ./src/messages/MOSDMap.h:72 #28 MOSDMap::~MOSDMap (this=0x55b4d7a52580, __in_chrg=<optimized out>) at ./src/messages/MOSDMap.h:72 #29 0x000055b4c867c744 in RefCountedObject::put (this=0x55b4d7a52580) at ./src/common/RefCountedObj.h:61 #30 0x000055b4c8637859 in Context::complete (this=0x55b4d8254ba0, r=<optimized out>) at ./src/include/Context.h:77 #31 0x000055b4c8bd2cbf in Finisher::finisher_thread_entry() () at ./src/common/Finisher.cc:67 #32 0x00007f00deb4d6db in start_thread (arg=0x7f00cac27700) at pthread_create.c:463 #33 0x00007f00dd8e888f in clone () at ../sysdeps/unix/sysv/linux/x86_64/clone.S:95
#2 Updated by Sage Weil about 4 years ago
Thread 1 (Thread 0x7f4a5cf43700 (LWP 39027)): #0 0x00007f4a75188bcb in tcmalloc::ThreadCache::ReleaseToCentralCache(tcmalloc::ThreadCache::FreeList*, unsigned long, int) () from /usr/lib/x86_64-linux-gnu/libtcmalloc.so.4 #1 0x00007f4a75188c9b in tcmalloc::ThreadCache::ListTooLong(tcmalloc::ThreadCache::FreeList*, unsigned long) () from /usr/lib/x86_64-linux-gnu/libtcmalloc.so.4 #2 0x00007f4a751976f5 in tc_free () from /usr/lib/x86_64-linux-gnu/libtcmalloc.so.4 #3 0x0000557c1a21ed46 in ceph::BackTrace::print(std::ostream&) const () #4 0x0000557c1a213ae6 in ?? () #5 <signal handler called> #6 0x00007f4a75188bcb in tcmalloc::ThreadCache::ReleaseToCentralCache(tcmalloc::ThreadCache::FreeList*, unsigned long, int) () from /usr/lib/x86_64-linux-gnu/libtcmalloc.so.4 #7 0x00007f4a75188c9b in tcmalloc::ThreadCache::ListTooLong(tcmalloc::ThreadCache::FreeList*, unsigned long) () from /usr/lib/x86_64-linux-gnu/libtcmalloc.so.4 #8 0x00007f4a751976f5 in tc_free () from /usr/lib/x86_64-linux-gnu/libtcmalloc.so.4 #9 0x0000557c1a092618 in std::_Rb_tree<unsigned long, std::pair<unsigned long const, ceph::buffer::v14_2_0::list>, std::_Select1st<std::pair<unsigned long const, ceph::buffer::v14_2_0::list> >, std::less<unsigned long>, std::allocator<std::pair<unsigned long const, ceph::buffer::v14_2_0::list> > >::_M_erase(std::_Rb_tree_node<std::pair<unsigned long const, ceph::buffer::v14_2_0::list> >*) () #10 0x0000557c1a0f4b4d in BlueStore::_read_cache(boost::intrusive_ptr<BlueStore::Onode>, unsigned long, unsigned long, int, std::map<unsigned long, ceph::buffer::v14_2_0::list, std::less<unsigned long>, std::allocator<std::pair<unsigned long const, ceph::buffer::v14_2_0::list> > >&, std::map<boost::intrusive_ptr<BlueStore::Blob>, std::__cxx11::list<BlueStore::read_req_t, std::allocator<BlueStore::read_req_t> >, std::less<boost::intrusive_ptr<BlueStore::Blob> >, std::allocator<std::pair<boost::intrusive_ptr<BlueStore::Blob> const, std::__cxx11::list<BlueStore::read_req_t, std::allocator<BlueStore::read_req_t> > > > >&) () #11 0x0000557c1a1012ab in BlueStore::_do_read(BlueStore::Collection*, boost::intrusive_ptr<BlueStore::Onode>, unsigned long, unsigned long, ceph::buffer::v14_2_0::list&, unsigned int, unsigned long) () #12 0x0000557c1a101ffd in BlueStore::read(boost::intrusive_ptr<ObjectStore::CollectionImpl>&, ghobject_t const&, unsigned long, unsigned long, ceph::buffer::v14_2_0::list&, unsigned int) () #13 0x0000557c19c2876e in OSDService::_get_map_bl(unsigned int, ceph::buffer::v14_2_0::list&) () #14 0x0000557c19c2b571 in OSDService::try_get_map(unsigned int) () #15 0x0000557c19c80b24 in OSD::_committed_osd_maps(unsigned int, unsigned int, MOSDMap*) () #16 0x0000557c19cd1737 in C_OnMapCommit::finish(int) () #17 0x0000557c19c88fe9 in Context::complete(int) () #18 0x0000557c1a22fd9d in Finisher::finisher_thread_entry() () #19 0x00007f4a746626db in start_thread (arg=0x7f4a5cf43700) at pthread_create.c:463 #20 0x00007f4a7340288f in clone () at ../sysdeps/unix/sysv/linux/x86_64/clone.S:95
/a/sage-2020-01-24_01:55:08-rados-wip-sage4-testing-2020-01-23-1347-distro-basic-smithi/4697860