Project

General

Profile

Bug #24974

Segmentation fault in tcmalloc::ThreadCache::ReleaseToCentralCache()

Added by Neha Ojha over 5 years ago. Updated about 4 years ago.

Status:
New
Priority:
Normal
Assignee:
-
Category:
-
Target version:
-
% Done:

0%

Source:
Tags:
Backport:
Regression:
No
Severity:
3 - minor
Reviewed:
Affected Versions:
ceph-qa-suite:
Component(RADOS):
Pull request ID:
Crash signature (v1):
Crash signature (v2):

Description

#0  0x00007fe1d6e0a0b3 in tcmalloc::ThreadCache::ReleaseToCentralCache(tcmalloc::ThreadCache::FreeList*, unsigned long, int) () from /usr/lib/libtcmalloc.so.4
[Current thread is 1 (Thread 0x7fe1d1687700 (LWP 13962))]
(gdb) bt
#0  0x00007fe1d6e0a0b3 in tcmalloc::ThreadCache::ReleaseToCentralCache(tcmalloc::ThreadCache::FreeList*, unsigned long, int) () from /usr/lib/libtcmalloc.so.4
#1  0x00007fe1d6e0a16b in tcmalloc::ThreadCache::ListTooLong(tcmalloc::ThreadCache::FreeList*, unsigned long) () from /usr/lib/libtcmalloc.so.4
#2  0x00007fe1d6e1c818 in tc_deletearray () from /usr/lib/libtcmalloc.so.4
#3  0x00007fe1d8231889 in __cxa_demangle () from /usr/lib/ceph/libceph-common.so.0
#4  0x00007fe1d7dad500 in ceph::BackTrace::print(std::ostream&) const () from /usr/lib/ceph/libceph-common.so.0
#5  0x00005589602b888a in handle_fatal_signal (signum=11) at /build/ceph-13.2.0-301-gbfc94b5/src/global/signal_handler.cc:119
#6  <signal handler called>
#7  0x00007fe1d6e0a0b3 in tcmalloc::ThreadCache::ReleaseToCentralCache(tcmalloc::ThreadCache::FreeList*, unsigned long, int) () from /usr/lib/libtcmalloc.so.4
#8  0x00007fe1d6e0a16b in tcmalloc::ThreadCache::ListTooLong(tcmalloc::ThreadCache::FreeList*, unsigned long) () from /usr/lib/libtcmalloc.so.4
#9  0x00007fe1d6e1c818 in tc_deletearray () from /usr/lib/libtcmalloc.so.4
#10 0x00007fe1d7d95a07 in std::__cxx11::_List_base<ceph::buffer::ptr, std::allocator<ceph::buffer::ptr> >::_M_clear() () from /usr/lib/ceph/libceph-common.so.0
#11 0x00007fe1d7f1f48b in PosixConnectedSocketImpl::send(ceph::buffer::list&, bool) () from /usr/lib/ceph/libceph-common.so.0
#12 0x00007fe1d7ef84b6 in AsyncConnection::_try_send(bool) () from /usr/lib/ceph/libceph-common.so.0
#13 0x00007fe1d7ef9b3e in AsyncConnection::write_message(Message*, ceph::buffer::list&, bool) () from /usr/lib/ceph/libceph-common.so.0
#14 0x00007fe1d7eff389 in AsyncConnection::handle_write() () from /usr/lib/ceph/libceph-common.so.0
#15 0x00007fe1d7f1a077 in EventCenter::process_events(unsigned int, std::chrono::duration<unsigned long, std::ratio<1l, 1000000000l> >*) () from /usr/lib/ceph/libceph-common.so.0
#16 0x00007fe1d7f1c9dc in ?? () from /usr/lib/ceph/libceph-common.so.0
#17 0x00007fe1d825258f in ?? () from /usr/lib/ceph/libceph-common.so.0
#18 0x00007fe1d64696ba in start_thread (arg=0x7fe1d1687700) at pthread_create.c:333
#19 0x00007fe1d5a7841d in clone () at ../sysdeps/unix/sysv/linux/x86_64/clone.S:109

/a/yuriw-2018-07-16_17:20:11-rados-wip-yuri4-testing-2018-07-16-1528-mimic-distro-basic-smithi/2788760/

History

#1 Updated by David Zafman almost 5 years ago

dzafman-2019-05-09_20:06:24-rados-wip-zafman-testing-distro-basic-smithi/3943901

Core was generated by `ceph-osd -f --cluster ceph -i 6'.
Program terminated with signal SIGSEGV, Segmentation fault.
#0  0x00007f00e0a50bcb in tcmalloc::ThreadCache::ReleaseToCentralCache(tcmalloc::ThreadCache::FreeList*, unsigned long, int) () from /usr/lib/x86_64-linux-gnu/libtcmalloc.so.4
[Current thread is 1 (Thread 0x7f00cac27700 (LWP 11964))]
(gdb) bt
#0  0x00007f00e0a50bcb in tcmalloc::ThreadCache::ReleaseToCentralCache(tcmalloc::ThreadCache::FreeList*, unsigned long, int) () from /usr/lib/x86_64-linux-gnu/libtcmalloc.so.4
#1  0x00007f00e0a50c9b in tcmalloc::ThreadCache::ListTooLong(tcmalloc::ThreadCache::FreeList*, unsigned long) () from /usr/lib/x86_64-linux-gnu/libtcmalloc.so.4
#2  0x00007f00e0a5f6f5 in tc_free () from /usr/lib/x86_64-linux-gnu/libtcmalloc.so.4
#3  0x000055b4c8bbf317 in __gnu_cxx::new_allocator<char>::deallocate (this=0x55b4d4fe6e88, __p=<optimized out>) at /usr/include/c++/8/ext/new_allocator.h:116
#4  std::allocator_traits<std::allocator<char> >::deallocate (__a=..., __n=<optimized out>, __p=<optimized out>) at /usr/include/c++/8/bits/alloc_traits.h:462
#5  std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> >::_M_destroy (__size=<optimized out>, this=0x55b4d4fe6e88)
    at /usr/include/c++/8/bits/basic_string.h:226
#6  std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> >::_M_dispose (this=0x55b4d4fe6e88) at /usr/include/c++/8/bits/basic_string.h:221
#7  std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> >::~basic_string (this=0x55b4d4fe6e88, __in_chrg=<optimized out>)
    at /usr/include/c++/8/bits/basic_string.h:647
#8  pidfh::~pidfh (this=0x55b4d4fe6e80, __in_chrg=<optimized out>) at ./src/global/pidfile.cc:52
#9  pidfile_remove () at ./src/global/pidfile.cc:210
#10 0x000055b4c8bc17c1 in handle_fatal_signal(int) () at ./src/global/signal_handler.cc:162
#11 <signal handler called>
#12 0x00007f00e0a50bcb in tcmalloc::ThreadCache::ReleaseToCentralCache(tcmalloc::ThreadCache::FreeList*, unsigned long, int) () from /usr/lib/x86_64-linux-gnu/libtcmalloc.so.4
#13 0x00007f00e0a50c9b in tcmalloc::ThreadCache::ListTooLong(tcmalloc::ThreadCache::FreeList*, unsigned long) () from /usr/lib/x86_64-linux-gnu/libtcmalloc.so.4
#14 0x00007f00e0a5f6f5 in tc_free () from /usr/lib/x86_64-linux-gnu/libtcmalloc.so.4
#15 0x000055b4c864cd65 in ceph::buffer::v14_2_0::ptr_node::disposer::operator() (this=<optimized out>, delete_this=0x55b4d78d34e0) at ./src/include/buffer.h:401
#16 ceph::buffer::v14_2_0::list::buffers_t::clear_and_dispose (this=0x55b4db3e8558) at ./src/include/buffer.h:641
#17 ceph::buffer::v14_2_0::list::~list (this=0x55b4db3e8558, __in_chrg=<optimized out>) at ./src/include/buffer.h:990
#18 std::pair<unsigned int const, ceph::buffer::v14_2_0::list>::~pair (this=0x55b4db3e8550, __in_chrg=<optimized out>) at /usr/include/c++/8/bits/stl_pair.h:198
#19 __gnu_cxx::new_allocator<std::_Rb_tree_node<std::pair<unsigned int const, ceph::buffer::v14_2_0::list> > >::destroy<std::pair<unsigned int const, ceph::buffer::v14_2_0::list> >
    (this=0x55b4d7a52778, __p=0x55b4db3e8550) at /usr/include/c++/8/ext/new_allocator.h:140
#20 std::allocator_traits<std::allocator<std::_Rb_tree_node<std::pair<unsigned int const, ceph::buffer::v14_2_0::list> > > >::destroy<std::pair<unsigned int const, ceph::buffer::v14_2_0::list> > (__a=..., __p=0x55b4db3e8550) at /usr/include/c++/8/bits/alloc_traits.h:487
#21 std::_Rb_tree<unsigned int, std::pair<unsigned int const, ceph::buffer::v14_2_0::list>, std::_Select1st<std::pair<unsigned int const, ceph::buffer::v14_2_0::list> >, std::less<unsigned int>, std::allocator<std::pair<unsigned int const, ceph::buffer::v14_2_0::list> > >::_M_destroy_node (this=0x55b4d7a52778, __p=0x55b4db3e8530)
    at /usr/include/c++/8/bits/stl_tree.h:661
#22 std::_Rb_tree<unsigned int, std::pair<unsigned int const, ceph::buffer::v14_2_0::list>, std::_Select1st<std::pair<unsigned int const, ceph::buffer::v14_2_0::list> >, std::less<unsigned int>, std::allocator<std::pair<unsigned int const, ceph::buffer::v14_2_0::list> > >::_M_drop_node (this=0x55b4d7a52778, __p=0x55b4db3e8530)
    at /usr/include/c++/8/bits/stl_tree.h:669
#23 std::_Rb_tree<unsigned int, std::pair<unsigned int const, ceph::buffer::v14_2_0::list>, std::_Select1st<std::pair<unsigned int const, ceph::buffer::v14_2_0::list> >, std::less<unsigned int>, std::allocator<std::pair<unsigned int const, ceph::buffer::v14_2_0::list> > >::_M_erase (this=this@entry=0x55b4d7a52778, __x=0x55b4db3e8530)
    at /usr/include/c++/8/bits/stl_tree.h:1874
#24 0x000055b4c864cd2d in std::_Rb_tree<unsigned int, std::pair<unsigned int const, ceph::buffer::v14_2_0::list>, std::_Select1st<std::pair<unsigned int const, ceph::buffer::v14_2_0::list> >, std::less<unsigned int>, std::allocator<std::pair<unsigned int const, ceph::buffer::v14_2_0::list> > >::_M_erase (this=this@entry=0x55b4d7a52778, __x=0x55b4db3e6e70)
    at /usr/include/c++/8/bits/stl_tree.h:1866
#25 0x000055b4c867dc69 in std::_Rb_tree<unsigned int, std::pair<unsigned int const, ceph::buffer::v14_2_0::list>, std::_Select1st<std::pair<unsigned int const, ceph::buffer::v14_2_0::list> >, std::less<unsigned int>, std::allocator<std::pair<unsigned int const, ceph::buffer::v14_2_0::list> > >::~_Rb_tree (this=0x55b4d7a52778, __in_chrg=<optimized out>)
    at /usr/include/c++/8/bits/stl_tree.h:964
#26 std::map<unsigned int, ceph::buffer::v14_2_0::list, std::less<unsigned int>, std::allocator<std::pair<unsigned int const, ceph::buffer::v14_2_0::list> > >::~map (
    this=0x55b4d7a52778, __in_chrg=<optimized out>) at /usr/include/c++/8/bits/stl_map.h:300
#27 MOSDMap::~MOSDMap (this=0x55b4d7a52580, __in_chrg=<optimized out>) at ./src/messages/MOSDMap.h:72
#28 MOSDMap::~MOSDMap (this=0x55b4d7a52580, __in_chrg=<optimized out>) at ./src/messages/MOSDMap.h:72
#29 0x000055b4c867c744 in RefCountedObject::put (this=0x55b4d7a52580) at ./src/common/RefCountedObj.h:61
#30 0x000055b4c8637859 in Context::complete (this=0x55b4d8254ba0, r=<optimized out>) at ./src/include/Context.h:77
#31 0x000055b4c8bd2cbf in Finisher::finisher_thread_entry() () at ./src/common/Finisher.cc:67
#32 0x00007f00deb4d6db in start_thread (arg=0x7f00cac27700) at pthread_create.c:463
#33 0x00007f00dd8e888f in clone () at ../sysdeps/unix/sysv/linux/x86_64/clone.S:95

#2 Updated by Sage Weil about 4 years ago

Thread 1 (Thread 0x7f4a5cf43700 (LWP 39027)):
#0  0x00007f4a75188bcb in tcmalloc::ThreadCache::ReleaseToCentralCache(tcmalloc::ThreadCache::FreeList*, unsigned long, int) () from /usr/lib/x86_64-linux-gnu/libtcmalloc.so.4
#1  0x00007f4a75188c9b in tcmalloc::ThreadCache::ListTooLong(tcmalloc::ThreadCache::FreeList*, unsigned long) () from /usr/lib/x86_64-linux-gnu/libtcmalloc.so.4
#2  0x00007f4a751976f5 in tc_free () from /usr/lib/x86_64-linux-gnu/libtcmalloc.so.4
#3  0x0000557c1a21ed46 in ceph::BackTrace::print(std::ostream&) const ()
#4  0x0000557c1a213ae6 in ?? ()
#5  <signal handler called>
#6  0x00007f4a75188bcb in tcmalloc::ThreadCache::ReleaseToCentralCache(tcmalloc::ThreadCache::FreeList*, unsigned long, int) () from /usr/lib/x86_64-linux-gnu/libtcmalloc.so.4
#7  0x00007f4a75188c9b in tcmalloc::ThreadCache::ListTooLong(tcmalloc::ThreadCache::FreeList*, unsigned long) () from /usr/lib/x86_64-linux-gnu/libtcmalloc.so.4
#8  0x00007f4a751976f5 in tc_free () from /usr/lib/x86_64-linux-gnu/libtcmalloc.so.4
#9  0x0000557c1a092618 in std::_Rb_tree<unsigned long, std::pair<unsigned long const, ceph::buffer::v14_2_0::list>, std::_Select1st<std::pair<unsigned long const, ceph::buffer::v14_2_0::list> >, std::less<unsigned long>, std::allocator<std::pair<unsigned long const, ceph::buffer::v14_2_0::list> > >::_M_erase(std::_Rb_tree_node<std::pair<unsigned long const, ceph::buffer::v14_2_0::list> >*) ()
#10 0x0000557c1a0f4b4d in BlueStore::_read_cache(boost::intrusive_ptr<BlueStore::Onode>, unsigned long, unsigned long, int, std::map<unsigned long, ceph::buffer::v14_2_0::list, std::less<unsigned long>, std::allocator<std::pair<unsigned long const, ceph::buffer::v14_2_0::list> > >&, std::map<boost::intrusive_ptr<BlueStore::Blob>, std::__cxx11::list<BlueStore::read_req_t, std::allocator<BlueStore::read_req_t> >, std::less<boost::intrusive_ptr<BlueStore::Blob> >, std::allocator<std::pair<boost::intrusive_ptr<BlueStore::Blob> const, std::__cxx11::list<BlueStore::read_req_t, std::allocator<BlueStore::read_req_t> > > > >&) ()
#11 0x0000557c1a1012ab in BlueStore::_do_read(BlueStore::Collection*, boost::intrusive_ptr<BlueStore::Onode>, unsigned long, unsigned long, ceph::buffer::v14_2_0::list&, unsigned int, unsigned long) ()
#12 0x0000557c1a101ffd in BlueStore::read(boost::intrusive_ptr<ObjectStore::CollectionImpl>&, ghobject_t const&, unsigned long, unsigned long, ceph::buffer::v14_2_0::list&, unsigned int) ()
#13 0x0000557c19c2876e in OSDService::_get_map_bl(unsigned int, ceph::buffer::v14_2_0::list&) ()
#14 0x0000557c19c2b571 in OSDService::try_get_map(unsigned int) ()
#15 0x0000557c19c80b24 in OSD::_committed_osd_maps(unsigned int, unsigned int, MOSDMap*) ()
#16 0x0000557c19cd1737 in C_OnMapCommit::finish(int) ()
#17 0x0000557c19c88fe9 in Context::complete(int) ()
#18 0x0000557c1a22fd9d in Finisher::finisher_thread_entry() ()
#19 0x00007f4a746626db in start_thread (arg=0x7f4a5cf43700) at pthread_create.c:463
#20 0x00007f4a7340288f in clone () at ../sysdeps/unix/sysv/linux/x86_64/clone.S:95

/a/sage-2020-01-24_01:55:08-rados-wip-sage4-testing-2020-01-23-1347-distro-basic-smithi/4697860

Also available in: Atom PDF