Bug #9840
closedMonitor hung when add new osd
0%
Description
ceph version: 0.80.6
Platform: Redhat RHLS 6.5
we want to test the replace disk case,
operator step:
1. ceph osd set noout
2. service ceph stop osd.2
3. rm data/osd-2/* -rf
4. ceph-osd -i 2 --mkfs
5. ceph osd rm 2
6. ceph osd create
7. service ceph start osd.2
then, master monitor hung, and the stack is?
#0 0x00000030638f82ce in __lll_lock_wait_private () from /lib64/libc.so.6
#1 0x000000306387d44b in _L_lock_9495 () from /lib64/libc.so.6
#2 0x000000306387a986 in malloc () from /lib64/libc.so.6
#3 0x000000306300cbad in _dl_map_object_deps () from /lib64/ld-linux-x86-64.so.2
#4 0x0000003063012aa1 in dl_open_worker () from /lib64/ld-linux-x86-64.so.2
#5 0x000000306300e1b6 in _dl_catch_error () from /lib64/ld-linux-x86-64.so.2
#6 0x00000030630124fa in _dl_open () from /lib64/ld-linux-x86-64.so.2
#7 0x0000003063926e10 in do_dlopen () from /lib64/libc.so.6
#8 0x000000306300e1b6 in _dl_catch_error () from /lib64/ld-linux-x86-64.so.2
#9 0x0000003063926f67 in __libc_dlopen_mode () from /lib64/libc.so.6
#10 0x00000030638feb25 in init () from /lib64/libc.so.6
#11 0x0000003063c0cd33 in pthread_once () from /lib64/libpthread.so.0
#12 0x00000030638fec54 in backtrace () from /lib64/libc.so.6
#13 0x0000000000fb3087 in ceph::BackTrace::BackTrace (this=0x7fce66deac90, s=0) at common/BackTrace.h:19
#14 0x000000000123b4e8 in handle_fatal_signal (signum=11) at global/signal_handler.cc:90
#15 <signal handler called>
#16 0x0000003063879a3d in _int_malloc () from /lib64/libc.so.6
#17 0x000000306387a991 in malloc () from /lib64/libc.so.6
#18 0x00007fceef0dc09d in operator new(unsigned long) () from /usr/lib64/libstdc++.so.6
#19 0x0000000000ee9e30 in __gnu_cxx::new_allocator<std::_Rb_tree_node<std::pair<pg_t const, pg_stat_t> > >::allocate (this=0x7fcec856b430, __n=1)
at /usr/lib/gcc/x86_64-redhat-linux/4.4.7/../../../../include/c++/4.4.7/ext/new_allocator.h:89
#20 0x0000000000ee954c in std::_Rb_tree<pg_t, std::pair<pg_t const, pg_stat_t>, std::_Select1st<std::pair<pg_t const, pg_stat_t> >, std::less<pg_t>, std::allocator<std::pair<pg_t const, pg_stat_t> > >::_M_get_node (this=0x7fcec856b430)
at /usr/lib/gcc/x86_64-redhat-linux/4.4.7/../../../../include/c++/4.4.7/bits/stl_tree.h:359
#21 0x0000000000ee865f in std::_Rb_tree<pg_t, std::pair<pg_t const, pg_stat_t>, std::_Select1st<std::pair<pg_t const, pg_stat_t> >, std::less<pg_t>, std::allocator<std::pair<pg_t const, pg_stat_t> > >::_M_create_node (
this=0x7fcec856b430, __x=...) at /usr/lib/gcc/x86_64-redhat-linux/4.4.7/../../../../include/c++/4.4.7/bits/stl_tree.h:369
#22 0x0000000000ee66bb in std::Rb_tree<pg_t, std::pair<pg_t const, pg_stat_t>, std::_Select1st<std::pair<pg_t const, pg_stat_t> >, std::less<pg_t>, std::allocator<std::pair<pg_t const, pg_stat_t> > >::_M_insert (this=0x7fcec856b430,
__x=0x0, __p=0x7fce62e07190, __v=...) at /usr/lib/gcc/x86_64-redhat-linux/4.4.7/../../../../include/c++/4.4.7/bits/stl_tree.h:881
#23 0x0000000000ee2117 in std::Rb_tree<pg_t, std::pair<pg_t const, pg_stat_t>, std::_Select1st<std::pair<pg_t const, pg_stat_t> >, std::less<pg_t>, std::allocator<std::pair<pg_t const, pg_stat_t> > >::_M_insert_unique (
this=0x7fcec856b430, __position=..., __v=...) at /usr/lib/gcc/x86_64-redhat-linux/4.4.7/../../../../include/c++/4.4.7/bits/stl_tree.h:1215
#24 0x0000000000edc6f8 in std::map<pg_t, pg_stat_t, std::less<pg_t>, std::allocator<std::pair<pg_t const, pg_stat_t> > >::insert (this=0x7fcec856b430, __position=..., __x=...)
at /usr/lib/gcc/x86_64-redhat-linux/4.4.7/../../../../include/c++/4.4.7/bits/stl_map.h:540
#25 0x0000000000ed680d in std::map<pg_t, pg_stat_t, std::less<pg_t>, std::allocator<std::pair<pg_t const, pg_stat_t> > >::operator[] (this=0x7fcec856b430, __k=...)
at /usr/lib/gcc/x86_64-redhat-linux/4.4.7/../../../../include/c++/4.4.7/bits/stl_map.h:450
#26 0x0000000001096529 in decode<pg_t, pg_stat_t> (m=std::map with 12 elements =
{...}
, p=...) at ./include/encoding.h:590
#27 0x0000000001078ffc in MPGStats::decode_payload (this=0x7fcec856b270) at ./messages/MPGStats.h:59
#28 0x00000000010769bf in decode_message (cct=0x3143d20, header=..., footer=..., front=..., middle=..., data=...) at msg/Message.cc:714
#29 0x000000000113ef49 in Pipe::read_message (this=0x7fced0000b90, pm=0x7fce66decb28, auth_handler=0x7fcec8810760) at msg/Pipe.cc:1954
#30 0x0000000001139fc2 in Pipe::reader (this=0x7fced0000b90) at msg/Pipe.cc:1483
#31 0x0000000001141b40 in Pipe::Reader::entry (this=0x7fced0000ba0) at msg/Pipe.h:49
#32 0x0000000000ff9317 in Thread::entry_wrapper (this=0x7fced0000ba0) at common/Thread.cc:60
#33 0x0000000000ff928e in Thread::_entry_func (arg=0x7fced0000ba0) at common/Thread.cc:45
#34 0x0000003063c079d1 in start_thread () from /lib64/libpthread.so.0
#35 0x00000030638e8b6d in clone () from /lib64/libc.so.6
Updated by qiu shanggao over 9 years ago
try again, monitor hung still
Thread 25 (Thread 0x7f93e5ec0700 (LWP 13652)):
#0 0x0000003168a0b5bc in pthread_cond_wait@GLIBC_2.3.2 () from /lib64/libpthread.so.0@GLIBC_2.3.2 () from /lib64/libpthread.so.0
#1 0x0000000000742bdb in ceph::log::Log::entry() ()
#2 0x0000003168a079d1 in start_thread () from /lib64/libpthread.so.0
#3 0x00000031686e8b6d in clone () from /lib64/libc.so.6
Thread 24 (Thread 0x7f93e527d700 (LWP 13653)):
#0 0x0000003168a0b98e in pthread_cond_timedwait
#1 0x00000000006eb864 in CephContextServiceThread::entry() ()
#2 0x0000003168a079d1 in start_thread () from /lib64/libpthread.so.0
#3 0x00000031686e8b6d in clone () from /lib64/libc.so.6
Thread 23 (Thread 0x7f93dffff700 (LWP 13654)):
#0 0x00000031686df343 in poll () from /lib64/libc.so.6
#1 0x000000000070038a in AdminSocket::entry() ()
#2 0x0000003168a079d1 in start_thread () from /lib64/libpthread.so.0
#3 0x00000031686e8b6d in clone () from /lib64/libc.so.6
Thread 22 (Thread 0x7f93df5fe700 (LWP 13655)):
#0 0x0000003168a0b5bc in pthread_cond_wait@GLIBC_2.3.2 () from /lib64/libpthread.so.0@GLIBC_2.3.2 () from /lib64/libpthread.so.0
#1 0x00000000008153d1 in SimpleMessenger::reaper_entry() ()
#2 0x000000000081bcdd in SimpleMessenger::ReaperThread::entry() ()
#3 0x0000003168a079d1 in start_thread () from /lib64/libpthread.so.0
#4 0x00000031686e8b6d in clone () from /lib64/libc.so.6
Thread 21 (Thread 0x7f93debfd700 (LWP 13656)):
#0 0x0000003168a0e264 in __lll_lock_wait () from /lib64/libpthread.so.0
#1 0x0000003168a09508 in _L_lock_854 () from /lib64/libpthread.so.0
#2 0x0000003168a093d7 in pthread_mutex_lock () from /lib64/libpthread.so.0
#3 0x0000000000740ce1 in Mutex::Lock(bool) ()
#4 0x0000000000816cb9 in SimpleMessenger::submit_message(Message*, Connection*, entity_addr_t const&, int, bool) ()
#5 0x0000000000817de9 in SimpleMessenger::_send_message(Message*, entity_inst_t const&, bool) ()
#6 0x00000000005a4ec0 in Paxos::extend_lease() ()
#7 0x00000000005871d9 in Context::complete(int) ()
#8 0x000000000081e9a3 in SafeTimer::timer_thread() ()
#9 0x0000000000820b5d in SafeTimerThread::entry() ()
#10 0x0000003168a079d1 in start_thread () from /lib64/libpthread.so.0
#11 0x00000031686e8b6d in clone () from /lib64/libc.so.6
Thread 20 (Thread 0x7f93de1fc700 (LWP 13657)):
#0 0x0000003168a0c392 in ?? () from /lib64/libpthread.so.0
#1 0x0000000000741972 in intrusive_ptr_release(RefCountedObject*) ()
#2 0x0000000000819d06 in DispatchQueue::QueueItem::~QueueItem() ()
#3 0x00000000008744aa in DispatchQueue::entry() ()
#4 0x0000000000819b3d in DispatchQueue::DispatchThread::entry() ()
#5 0x0000003168a079d1 in start_thread () from /lib64/libpthread.so.0
#6 0x00000031686e8b6d in clone () from /lib64/libc.so.6
Thread 19 (Thread 0x7f93dd7fb700 (LWP 13658)):
#0 0x0000003168a0e264 in __lll_lock_wait () from /lib64/libpthread.so.0
#1 0x0000003168a09508 in _L_lock_854 () from /lib64/libpthread.so.0
#2 0x0000003168a093d7 in pthread_mutex_lock () from /lib64/libpthread.so.0
#3 0x0000000000740ce1 in Mutex::Lock(bool) ()
#4 0x0000000000811671 in SimpleMessenger::add_accept_pipe(int) ()
#5 0x0000000000852902 in Accepter::entry() ()
#6 0x0000003168a079d1 in start_thread () from /lib64/libpthread.so.0
#7 0x00000031686e8b6d in clone () from /lib64/libc.so.6
Thread 18 (Thread 0x7f93e4280700 (LWP 13659)):
#0 0x0000003168a0e264 in __lll_lock_wait () from /lib64/libpthread.so.0
#1 0x0000003168a1102d in _L_cond_lock_886 () from /lib64/libpthread.so.0
#2 0x0000003168a10f07 in __pthread_mutex_cond_lock () from /lib64/libpthread.so.0
#3 0x0000003168a0b652 in pthread_cond_wait
#4 0x0000000000835a71 in Pipe::writer() ()
#5 0x000000000084145d in Pipe::Writer::entry() ()
#6 0x0000003168a079d1 in start_thread () from /lib64/libpthread.so.0
#7 0x00000031686e8b6d in clone () from /lib64/libc.so.6
Thread 17 (Thread 0x7f93e417f700 (LWP 13660)):
#0 0x0000003168a0e264 in __lll_lock_wait () from /lib64/libpthread.so.0
#1 0x0000003168a1102d in _L_cond_lock_886 () from /lib64/libpthread.so.0
#2 0x0000003168a10f07 in __pthread_mutex_cond_lock () from /lib64/libpthread.so.0
#3 0x0000003168a0b652 in pthread_cond_wait@GLIBC_2.3.2 () from /lib64/libpthread.so.0@GLIBC_2.3.2 () from /lib64/libpthread.so.0
#4 0x0000000000835a71 in Pipe::writer() ()
#5 0x000000000084145d in Pipe::Writer::entry() ()
#6 0x0000003168a079d1 in start_thread () from /lib64/libpthread.so.0
#7 0x00000031686e8b6d in clone () from /lib64/libc.so.6
Thread 16 (Thread 0x7f93dcdfa700 (LWP 13662)):
#0 0x00000031686df343 in poll () from /lib64/libc.so.6
#1 0x000000000087aff6 in SignalHandler::entry() ()
#2 0x0000003168a079d1 in start_thread () from /lib64/libpthread.so.0
#3 0x00000031686e8b6d in clone () from /lib64/libc.so.6
Thread 15 (Thread 0x7f93dc3f9700 (LWP 13745)):
#0 0x0000003168a0e264 in __lll_lock_wait () from /lib64/libpthread.so.0
#1 0x0000003168a09508 in _L_lock_854 () from /lib64/libpthread.so.0
#2 0x0000003168a093d7 in pthread_mutex_lock () from /lib64/libpthread.so.0
#3 0x0000000000740ce1 in Mutex::Lock(bool) ()
#4 0x0000000000873741 in DispatchQueue::enqueue(Message*, int, unsigned long) ()
#5 0x000000000083eec7 in Pipe::reader() ()
#6 0x000000000084147d in Pipe::Reader::entry() ()
#7 0x0000003168a079d1 in start_thread () from /lib64/libpthread.so.0
#8 0x00000031686e8b6d in clone () from /lib64/libc.so.6
Thread 14 (Thread 0x7f93dc2f8700 (LWP 13832)):
#0 0x0000003168a0e264 in __lll_lock_wait () from /lib64/libpthread.so.0
#1 0x0000003168a09508 in _L_lock_854 () from /lib64/libpthread.so.0
#2 0x0000003168a093d7 in pthread_mutex_lock () from /lib64/libpthread.so.0
#3 0x0000000000740ce1 in Mutex::Lock(bool) ()
#4 0x0000000000873741 in DispatchQueue::enqueue(Message*, int, unsigned long) ()
#5 0x000000000083eec7 in Pipe::reader() ()
#6 0x000000000084147d in Pipe::Reader::entry() ()
#7 0x0000003168a079d1 in start_thread () from /lib64/libpthread.so.0
#8 0x00000031686e8b6d in clone () from /lib64/libc.so.6
Thread 13 (Thread 0x7f93dc1f7700 (LWP 14417)):
#0 0x0000003168a0e264 in __lll_lock_wait () from /lib64/libpthread.so.0
#1 0x0000003168a09508 in _L_lock_854 () from /lib64/libpthread.so.0
#2 0x0000003168a093d7 in pthread_mutex_lock () from /lib64/libpthread.so.0
#3 0x0000000000740ce1 in Mutex::Lock(bool) ()
#4 0x0000000000873741 in DispatchQueue::enqueue(Message*, int, unsigned long) ()
#5 0x000000000083eec7 in Pipe::reader() ()
#6 0x000000000084147d in Pipe::Reader::entry() ()
#7 0x0000003168a079d1 in start_thread () from /lib64/libpthread.so.0
#8 0x00000031686e8b6d in clone () from /lib64/libc.so.6
Thread 12 (Thread 0x7f93b7fff700 (LWP 14419)):
#0 0x0000003168a0e264 in __lll_lock_wait () from /lib64/libpthread.so.0
#1 0x0000003168a1102d in _L_cond_lock_886 () from /lib64/libpthread.so.0
#2 0x0000003168a10f07 in __pthread_mutex_cond_lock () from /lib64/libpthread.so.0
#3 0x0000003168a0b652 in pthread_cond_wait
#4 0x0000000000835a71 in Pipe::writer() ()
#5 0x000000000084145d in Pipe::Writer::entry() ()
#6 0x0000003168a079d1 in start_thread () from /lib64/libpthread.so.0
#7 0x00000031686e8b6d in clone () from /lib64/libc.so.6
Thread 11 (Thread 0x7f93b7afa700 (LWP 16227)):
#0 0x00000031686f82ce in __lll_lock_wait_private () from /lib64/libc.so.6
#1 0x000000316867d44b in _L_lock_9495 () from /lib64/libc.so.6
#2 0x000000316867a986 in malloc () from /lib64/libc.so.6
#3 0x00000031752bd09d in operator new(unsigned long) () from /usr/lib64/libstdc++.so.6
#4 0x0000000000718780 in ceph::buffer::create(unsigned int) ()
#5 0x000000000082e66a in Pipe::read_message(Message**, AuthSessionHandler*) ()
#6 0x000000000083df99 in Pipe::reader() ()
#7 0x000000000084147d in Pipe::Reader::entry() ()
#8 0x0000003168a079d1 in start_thread () from /lib64/libpthread.so.0
#9 0x00000031686e8b6d in clone () from /lib64/libc.so.6
Thread 10 (Thread 0x7f93b79f9700 (LWP 16229)):
#0 0x0000003168a0b5bc in pthread_cond_wait@GLIBC_2.3.2 () from /lib64/libpthread.so.0@GLIBC_2.3.2 () from /lib64/libpthread.so.0
#1 0x0000000000835a71 in Pipe::writer() ()
#2 0x000000000084145d in Pipe::Writer::entry() ()
#3 0x0000003168a079d1 in start_thread () from /lib64/libpthread.so.0
#4 0x00000031686e8b6d in clone () from /lib64/libc.so.6
Thread 9 (Thread 0x7f93b78f8700 (LWP 16274)):
#0 0x0000003168a0e264 in __lll_lock_wait () from /lib64/libpthread.so.0
#1 0x0000003168a09508 in _L_lock_854 () from /lib64/libpthread.so.0
#2 0x0000003168a093d7 in pthread_mutex_lock () from /lib64/libpthread.so.0
#3 0x0000000000740ce1 in Mutex::Lock(bool) ()
#4 0x0000000000873741 in DispatchQueue::enqueue(Message*, int, unsigned long) ()
#5 0x000000000083eec7 in Pipe::reader() ()
#6 0x000000000084147d in Pipe::Reader::entry() ()
#7 0x0000003168a079d1 in start_thread () from /lib64/libpthread.so.0
#8 0x00000031686e8b6d in clone () from /lib64/libc.so.6
Thread 8 (Thread 0x7f93b77f7700 (LWP 16275)):
#0 0x0000003168a0e264 in __lll_lock_wait () from /lib64/libpthread.so.0
#1 0x0000003168a1102d in _L_cond_lock_886 () from /lib64/libpthread.so.0
#2 0x0000003168a10f07 in __pthread_mutex_cond_lock () from /lib64/libpthread.so.0
#3 0x0000003168a0b652 in pthread_cond_wait
#4 0x0000000000835a71 in Pipe::writer() ()
#5 0x000000000084145d in Pipe::Writer::entry() ()
#6 0x0000003168a079d1 in start_thread () from /lib64/libpthread.so.0
#7 0x00000031686e8b6d in clone () from /lib64/libc.so.6
Thread 7 (Thread 0x7f93b74f4700 (LWP 20550)):
#0 0x0000003168a0b5bc in pthread_cond_wait@GLIBC_2.3.2 () from /lib64/libpthread.so.0@GLIBC_2.3.2 () from /lib64/libpthread.so.0
#1 0x00000000008569a5 in LevelDBStore::compact_thread_entry() ()
#2 0x000000000053bb3d in LevelDBStore::CompactThread::entry() ()
#3 0x0000003168a079d1 in start_thread () from /lib64/libpthread.so.0
#4 0x00000031686e8b6d in clone () from /lib64/libc.so.6
Thread 6 (Thread 0x7f93b6af3700 (LWP 20551)):
#0 0x0000003168a0b5bc in pthread_cond_wait
#1 0x00007f93e5f0e603 in leveldb::(anonymous namespace)::PosixEnv::BGThreadWrapper(void*) () from /root/release/lib/libleveldb.so.1
#2 0x0000003168a079d1 in start_thread () from /lib64/libpthread.so.0
#3 0x00000031686e8b6d in clone () from /lib64/libc.so.6
Thread 5 (Thread 0x7f93b7efe700 (LWP 20060)):
#0 0x0000003168a0e264 in __lll_lock_wait () from /lib64/libpthread.so.0
#1 0x0000003168a09508 in _L_lock_854 () from /lib64/libpthread.so.0
#2 0x0000003168a093d7 in pthread_mutex_lock () from /lib64/libpthread.so.0
#3 0x0000000000740ce1 in Mutex::Lock(bool) ()
#4 0x0000000000873741 in DispatchQueue::enqueue(Message*, int, unsigned long) ()
#5 0x000000000083eec7 in Pipe::reader() ()
#6 0x000000000084147d in Pipe::Reader::entry() ()
#7 0x0000003168a079d1 in start_thread () from /lib64/libpthread.so.0
#8 0x00000031686e8b6d in clone () from /lib64/libc.so.6
Thread 4 (Thread 0x7f93b7dfd700 (LWP 20062)):
#0 0x0000003168a0e264 in __lll_lock_wait () from /lib64/libpthread.so.0
#1 0x0000003168a1102d in _L_cond_lock_886 () from /lib64/libpthread.so.0
#2 0x0000003168a10f07 in __pthread_mutex_cond_lock () from /lib64/libpthread.so.0
#3 0x0000003168a0b652 in pthread_cond_wait@GLIBC_2.3.2 () from /lib64/libpthread.so.0@GLIBC_2.3.2 () from /lib64/libpthread.so.0
#4 0x0000000000835a71 in Pipe::writer() ()
#5 0x000000000084145d in Pipe::Writer::entry() ()
#6 0x0000003168a079d1 in start_thread () from /lib64/libpthread.so.0
#7 0x00000031686e8b6d in clone () from /lib64/libc.so.6
Thread 3 (Thread 0x7f93b7cfc700 (LWP 20459)):
#0 0x00000031686f82ce in __lll_lock_wait_private () from /lib64/libc.so.6
#1 0x000000316867d44b in _L_lock_9495 () from /lib64/libc.so.6
#2 0x000000316867a986 in malloc () from /lib64/libc.so.6
#3 0x00000031686fee19 in backtrace_symbols () from /lib64/libc.so.6
#4 0x000000000087a696 in handle_fatal_signal(int) ()
#5 <signal handler called>
#6 0x0000003168676239 in malloc_consolidate () from /lib64/libc.so.6
#7 0x0000003168679405 in _int_malloc () from /lib64/libc.so.6
#8 0x000000316867a991 in malloc () from /lib64/libc.so.6
#9 0x00000031686fee19 in backtrace_symbols () from /lib64/libc.so.6
#10 0x000000000087a696 in handle_fatal_signal(int) ()
#11 <signal handler called>
#12 0x0000003168632925 in raise () from /lib64/libc.so.6
#13 0x0000003168634105 in abort () from /lib64/libc.so.6
#14 0x0000003168670837 in __libc_message () from /lib64/libc.so.6
#15 0x0000003168676166 in malloc_printerr () from /lib64/libc.so.6
#16 0x0000003168679be4 in _int_malloc () from /lib64/libc.so.6
#17 0x000000316867a991 in malloc () from /lib64/libc.so.6
#18 0x00000031752bd09d in operator new(unsigned long) () from /usr/lib64/libstdc++.so.6
#19 0x0000000000718780 in ceph::buffer::create(unsigned int) ()
#20 0x000000000082e66a in Pipe::read_message(Message**, AuthSessionHandler*) ()
#21 0x000000000083df99 in Pipe::reader() ()
#22 0x000000000084147d in Pipe::Reader::entry() ()
#23 0x0000003168a079d1 in start_thread () from /lib64/libpthread.so.0
#24 0x00000031686e8b6d in clone () from /lib64/libc.so.6
Thread 2 (Thread 0x7f93b7bfb700 (LWP 20460)):
#0 0x0000003168a0b5bc in pthread_cond_wait
#1 0x0000000000835a71 in Pipe::writer() ()
#2 0x000000000084145d in Pipe::Writer::entry() ()
#3 0x0000003168a079d1 in start_thread () from /lib64/libpthread.so.0
#4 0x00000031686e8b6d in clone () from /lib64/libc.so.6
Thread 1 (Thread 0x7f93e5ec8760 (LWP 13651)):
#0 0x0000003168a0822d in pthread_join () from /lib64/libpthread.so.0
#1 0x000000000081be22 in Thread::join(void**) ()
#2 0x0000000000818960 in SimpleMessenger::wait() ()
#3 0x00000000005393f8 in main ()
Updated by qiu shanggao over 9 years ago
using the valgrin find errors:
17554 Thread 7:
17554 Invalid read of size 4
17554 at 0x3168A0C380: pthread_spin_lock (in /lib64/libpthread-2.12.so)
17554 by 0x741971: intrusive_ptr_release(RefCountedObject*) (Spinlock.h:45)
17554 by 0x819D05: DispatchQueue::QueueItem::~QueueItem() (intrusive_ptr.hpp:101)
17554 by 0x8744A9: DispatchQueue::entry() (DispatchQueue.cc:83)
17554 by 0x819B3C: DispatchQueue::DispatchThread::entry() (DispatchQueue.h:104)
17554 by 0x3168A079D0: start_thread (in /lib64/libpthread-2.12.so)
17554 by 0x31686E8B6C: clone (in /lib64/libc-2.12.so)
17554 Address 0xe6f0ca8 is 8 bytes inside a block of size 1,136 free'd
17554 at 0x4A05FD6: operator delete(void*) (vg_replace_malloc.c:480)
17554 by 0x5E703F: OSDMonitor::preprocess_boot(MOSDBoot*) (RefCountedObj.h:36)
17554 by 0x5F286A: OSDMonitor::preprocess_query(PaxosServiceMessage*) (OSDMonitor.cc:690)
17554 by 0x5B5984: PaxosService::dispatch(PaxosServiceMessage*) (PaxosService.cc:73)
17554 by 0x584F6A: Monitor::dispatch(MonSession*, Message*, bool) (Monitor.cc:3035)
17554 by 0x58576B: Monitor::_ms_dispatch(Message*) (Monitor.cc:2950)
17554 by 0x5A17C1: Monitor::ms_dispatch(Message*) (Monitor.h:750)
17554 by 0x874691: DispatchQueue::entry() (Messenger.h:587)
17554 by 0x819B3C: DispatchQueue::DispatchThread::entry() (DispatchQueue.h:104)
17554 by 0x3168A079D0: start_thread (in /lib64/libpthread-2.12.so)
17554 by 0x31686E8B6C: clone (in /lib64/libc-2.12.so)
17554
17554 Invalid read of size 4
17554 at 0x741972: intrusive_ptr_release(RefCountedObject*) (atomic.h:52)
17554 by 0x819D05: DispatchQueue::QueueItem::~QueueItem() (intrusive_ptr.hpp:101)
17554 by 0x8744A9: DispatchQueue::entry() (DispatchQueue.cc:83)
17554 by 0x819B3C: DispatchQueue::DispatchThread::entry() (DispatchQueue.h:104)
17554 by 0x3168A079D0: start_thread (in /lib64/libpthread-2.12.so)
17554 by 0x31686E8B6C: clone (in /lib64/libc-2.12.so)
17554 Address 0xe6f0cac is 12 bytes inside a block of size 1,136 free'd
17554 at 0x4A05FD6: operator delete(void*) (vg_replace_malloc.c:480)
17554 by 0x5E703F: OSDMonitor::preprocess_boot(MOSDBoot*) (RefCountedObj.h:36)
17554 by 0x5F286A: OSDMonitor::preprocess_query(PaxosServiceMessage*) (OSDMonitor.cc:690)
17554 by 0x5B5984: PaxosService::dispatch(PaxosServiceMessage*) (PaxosService.cc:73)
17554 by 0x584F6A: Monitor::dispatch(MonSession*, Message*, bool) (Monitor.cc:3035)
17554 by 0x58576B: Monitor::_ms_dispatch(Message*) (Monitor.cc:2950)
17554 by 0x5A17C1: Monitor::ms_dispatch(Message*) (Monitor.h:750)
17554 by 0x874691: DispatchQueue::entry() (Messenger.h:587)
17554 by 0x819B3C: DispatchQueue::DispatchThread::entry() (DispatchQueue.h:104)
17554 by 0x3168A079D0: start_thread (in /lib64/libpthread-2.12.so)
17554 by 0x31686E8B6C: clone (in /lib64/libc-2.12.so)
17554
17554 Invalid write of size 4
17554 at 0x74197B: intrusive_ptr_release(RefCountedObject*) (atomic.h:52)
17554 by 0x819D05: DispatchQueue::QueueItem::~QueueItem() (intrusive_ptr.hpp:101)
17554 by 0x8744A9: DispatchQueue::entry() (DispatchQueue.cc:83)
17554 by 0x819B3C: DispatchQueue::DispatchThread::entry() (DispatchQueue.h:104)
17554 by 0x3168A079D0: start_thread (in /lib64/libpthread-2.12.so)
17554 by 0x31686E8B6C: clone (in /lib64/libc-2.12.so)
17554 Address 0xe6f0cac is 12 bytes inside a block of size 1,136 free'd
17554 at 0x4A05FD6: operator delete(void*) (vg_replace_malloc.c:480)
17554 by 0x5E703F: OSDMonitor::preprocess_boot(MOSDBoot*) (RefCountedObj.h:36)
17554 by 0x5F286A: OSDMonitor::preprocess_query(PaxosServiceMessage*) (OSDMonitor.cc:690)
17554 by 0x5B5984: PaxosService::dispatch(PaxosServiceMessage*) (PaxosService.cc:73)
17554 by 0x584F6A: Monitor::dispatch(MonSession*, Message*, bool) (Monitor.cc:3035)
17554 by 0x58576B: Monitor::_ms_dispatch(Message*) (Monitor.cc:2950)
17554 by 0x5A17C1: Monitor::ms_dispatch(Message*) (Monitor.h:750)
17554 by 0x874691: DispatchQueue::entry() (Messenger.h:587)
17554 by 0x819B3C: DispatchQueue::DispatchThread::entry() (DispatchQueue.h:104)
17554 by 0x3168A079D0: start_thread (in /lib64/libpthread-2.12.so)
17554 by 0x31686E8B6C: clone (in /lib64/libc-2.12.so)
17554
17554 Invalid write of size 4
17554 at 0x3168A0C3C0: pthread_spin_init (in /lib64/libpthread-2.12.so)
17554 by 0x741982: intrusive_ptr_release(RefCountedObject*) (Spinlock.h:50)
17554 by 0x819D05: DispatchQueue::QueueItem::~QueueItem() (intrusive_ptr.hpp:101)
17554 by 0x8744A9: DispatchQueue::entry() (DispatchQueue.cc:83)
17554 by 0x819B3C: DispatchQueue::DispatchThread::entry() (DispatchQueue.h:104)
17554 by 0x3168A079D0: start_thread (in /lib64/libpthread-2.12.so)
17554 by 0x31686E8B6C: clone (in /lib64/libc-2.12.so)
17554 Address 0xe6f0ca8 is 8 bytes inside a block of size 1,136 free'd
17554 at 0x4A05FD6: operator delete(void*) (vg_replace_malloc.c:480)
17554 by 0x5E703F: OSDMonitor::preprocess_boot(MOSDBoot*) (RefCountedObj.h:36)
17554 by 0x5F286A: OSDMonitor::preprocess_query(PaxosServiceMessage*) (OSDMonitor.cc:690)
17554 by 0x5B5984: PaxosService::dispatch(PaxosServiceMessage*) (PaxosService.cc:73)
17554 by 0x584F6A: Monitor::dispatch(MonSession*, Message*, bool) (Monitor.cc:3035)
17554 by 0x58576B: Monitor::_ms_dispatch(Message*) (Monitor.cc:2950)
17554 by 0x5A17C1: Monitor::ms_dispatch(Message*) (Monitor.h:750)
17554 by 0x874691: DispatchQueue::entry() (Messenger.h:587)
17554 by 0x819B3C: DispatchQueue::DispatchThread::entry() (DispatchQueue.h:104)
17554 by 0x3168A079D0: start_thread (in /lib64/libpthread-2.12.so)
17554 by 0x31686E8B6C: clone (in /lib64/libc-2.12.so)
17554