Project

General

Profile

Bug #18042 » bt.txt

Sage Weil, 11/25/2016 02:36 PM

 
[New LWP 17032]
[New LWP 18878]
[New LWP 21174]
[New LWP 17364]
[Thread debugging using libthread_db enabled]
Using host libthread_db library "/lib64/libthread_db.so.1".
Core was generated by `ceph-osd -f --cluster ceph -i 0'.
Program terminated with signal 6, Aborted.
#0 0x00007fbb03e23fcb in raise () from /lib64/libpthread.so.0
Missing separate debuginfos, use: debuginfo-install boost-iostreams-1.53.0-25.el7.x86_64 boost-random-1.53.0-25.el7.x86_64 boost-system-1.53.0-25.el7.x86_64 boost-thread-1.53.0-25.el7.x86_64 bzip2-libs-1.0.6-13.el7.x86_64 fuse-libs-2.9.2-6.el7.x86_64 glibc-2.17-106.el7_2.8.x86_64 gperftools-libs-2.4-7.el7.x86_64 leveldb-1.12.0-11.el7.x86_64 libaio-0.3.109-13.el7.x86_64 libblkid-2.23.2-26.el7_2.3.x86_64 libgcc-4.8.5-4.el7.x86_64 libstdc++-4.8.5-4.el7.x86_64 libunwind-1.1-5.el7_2.2.x86_64 libuuid-2.23.2-26.el7_2.3.x86_64 lttng-ust-2.4.1-1.el7.x86_64 nspr-4.10.8-2.el7_1.x86_64 nss-3.19.1-18.el7.x86_64 nss-softokn-3.16.2.3-13.el7_1.x86_64 nss-softokn-freebl-3.16.2.3-13.el7_1.x86_64 nss-util-3.19.1-4.el7_1.x86_64 snappy-1.1.0-3.el7.x86_64 sqlite-3.7.17-8.el7.x86_64 userspace-rcu-0.7.16-1.el7.x86_64 zlib-1.2.7-15.el7.x86_64
(gdb) bt
#0 0x00007fbb03e23fcb in raise () from /lib64/libpthread.so.0
#1 0x00007fbb06006975 in reraise_fatal (signum=6) at global/signal_handler.cc:71
#2 handle_fatal_signal (signum=6) at global/signal_handler.cc:133
#3 <signal handler called>
#4 0x00007fbb023e75f7 in sigtimedwait () from /lib64/libc.so.6
#5 0x00007fbb001b95a8 in ?? ()
#6 0x0000000000000000 in ?? ()
(gdb) thr app all bt

Thread 87 (Thread 0x7fbad975d700 (LWP 17364)):
#0 0x00007fbb03e206d5 in pthread_cond_wait@@GLIBC_2.3.2 () from /lib64/libpthread.so.0
#1 0x00007fbb061f60b9 in Wait (mutex=..., this=0x7fbb1137dab8) at common/Cond.h:56
#2 Pipe::DelayedDelivery::entry (this=0x7fbb1137d9e0) at msg/simple/Pipe.cc:203
#3 0x00007fbb03e1cdc5 in start_thread () from /lib64/libpthread.so.0
#4 0x00007fbb024a828d in ?? () from /lib64/libc.so.6
#5 0x0000000000000000 in ?? ()

Thread 86 (Thread 0x7fbad7b56700 (LWP 21174)):
#0 0x00007fbb03e22f4d in __lll_lock_wait () from /lib64/libpthread.so.0
#1 0x00007fbb03e1ed02 in _L_lock_791 () from /lib64/libpthread.so.0
#2 0x00007fbb03e1ec08 in pthread_mutex_lock () from /lib64/libpthread.so.0
#3 0x00007fbb060b47d8 in Mutex::Lock (this=0x7fbb10fa44b0, no_lockdep=no_lockdep@entry=false) at common/Mutex.cc:110
#4 0x00007fbb061f8139 in Pipe::fault (this=this@entry=0x7fbb12d74800, onread=onread@entry=true) at msg/simple/Pipe.cc:1391
#5 0x00007fbb061fecc0 in Pipe::reader (this=0x7fbb12d74800) at msg/simple/Pipe.cc:1697
#6 0x00007fbb062084bd in Pipe::Reader::entry (this=<optimized out>) at msg/simple/Pipe.h:52
#7 0x00007fbb03e1cdc5 in start_thread () from /lib64/libpthread.so.0
#8 0x00007fbb024a828d in ?? () from /lib64/libc.so.6
#9 0x0000000000000000 in ?? ()

Thread 85 (Thread 0x7fbad8559700 (LWP 18878)):
#0 0x00007fbb03e22f4d in __lll_lock_wait () from /lib64/libpthread.so.0
#1 0x00007fbb03e1ed02 in _L_lock_791 () from /lib64/libpthread.so.0
#2 0x00007fbb03e1ec08 in pthread_mutex_lock () from /lib64/libpthread.so.0
#3 0x00007fbb060b47d8 in Mutex::Lock (this=0x7fbb10fa44b0, no_lockdep=no_lockdep@entry=false) at common/Mutex.cc:110
#4 0x00007fbb061f8139 in Pipe::fault (this=this@entry=0x7fbb112e3400, onread=onread@entry=true) at msg/simple/Pipe.cc:1391
#5 0x00007fbb061fecc0 in Pipe::reader (this=0x7fbb112e3400) at msg/simple/Pipe.cc:1697
#6 0x00007fbb062084bd in Pipe::Reader::entry (this=<optimized out>) at msg/simple/Pipe.h:52
#7 0x00007fbb03e1cdc5 in start_thread () from /lib64/libpthread.so.0
#8 0x00007fbb024a828d in ?? () from /lib64/libc.so.6
#9 0x0000000000000000 in ?? ()

Thread 84 (Thread 0x7fbb056b9700 (LWP 17032)):
#0 0x00007fbb03e206d5 in pthread_cond_wait@@GLIBC_2.3.2 () from /lib64/libpthread.so.0
#1 0x00007fbb062055cf in Wait (mutex=..., this=0x7fbb111bb248) at common/Cond.h:56
#2 Pipe::writer (this=0x7fbb111ba000) at msg/simple/Pipe.cc:1864
#3 0x00007fbb062084dd in Pipe::Writer::entry (this=<optimized out>) at msg/simple/Pipe.h:63
#4 0x00007fbb03e1cdc5 in start_thread () from /lib64/libpthread.so.0
#5 0x00007fbb024a828d in ?? () from /lib64/libc.so.6
#6 0x0000000000000000 in ?? ()

Thread 83 (Thread 0x7fbad8e5b700 (LWP 19777)):
#0 0x00007fbb03e206d5 in pthread_cond_wait@@GLIBC_2.3.2 () from /lib64/libpthread.so.0
#1 0x00007fbb061ff0ad in Wait (mutex=..., this=0x7fbb117bd248) at common/Cond.h:56
#2 Pipe::reader (this=0x7fbb117bc000) at msg/simple/Pipe.cc:1529
#3 0x00007fbb062084bd in Pipe::Reader::entry (this=<optimized out>) at msg/simple/Pipe.h:52
#4 0x00007fbb03e1cdc5 in start_thread () from /lib64/libpthread.so.0
#5 0x00007fbb024a828d in ?? () from /lib64/libc.so.6
#6 0x0000000000000000 in ?? ()

Thread 82 (Thread 0x7fbadbe68700 (LWP 21078)):
#0 0x00007fbb03e206d5 in pthread_cond_wait@@GLIBC_2.3.2 () from /lib64/libpthread.so.0
#1 0x00007fbb062055cf in Wait (mutex=..., this=0x7fbb117be648) at common/Cond.h:56
#2 Pipe::writer (this=0x7fbb117bd400) at msg/simple/Pipe.cc:1864
#3 0x00007fbb062084dd in Pipe::Writer::entry (this=<optimized out>) at msg/simple/Pipe.h:63
#4 0x00007fbb03e1cdc5 in start_thread () from /lib64/libpthread.so.0
#5 0x00007fbb024a828d in ?? () from /lib64/libc.so.6
#6 0x0000000000000000 in ?? ()

Thread 81 (Thread 0x7fbae8c8e700 (LWP 17016)):
#0 0x00007fbb03e20a82 in pthread_cond_timedwait@@GLIBC_2.3.2 () from /lib64/libpthread.so.0
#1 0x00007fbb05a7a6e5 in WaitUntil (when=..., mutex=..., this=0x7fbb10edca58) at common/Cond.h:72
#2 WaitInterval (interval=..., mutex=..., cct=<optimized out>, this=0x7fbb10edca58) at common/Cond.h:81
#3 OSD::ShardedOpWQ::_process (this=0x7fbb1114b1e8, thread_index=<optimized out>, hb=0x7fbb111935d0) at osd/OSD.cc:8644
#4 0x00007fbb060f36e7 in ShardedThreadPool::shardedthreadpool_worker (this=0x7fbb1114a768, thread_index=<optimized out>) at common/WorkQueue.cc:340
#5 0x00007fbb060f5650 in ShardedThreadPool::WorkThreadSharded::entry (this=<optimized out>) at common/WorkQueue.h:684
#6 0x00007fbb03e1cdc5 in start_thread () from /lib64/libpthread.so.0
#7 0x00007fbb024a828d in ?? () from /lib64/libc.so.6
#8 0x0000000000000000 in ?? ()

Thread 80 (Thread 0x7fbada05f700 (LWP 17354)):
#0 0x00007fbb03e206d5 in pthread_cond_wait@@GLIBC_2.3.2 () from /lib64/libpthread.so.0
#1 0x00007fbb061f60b9 in Wait (mutex=..., this=0x7fbb1137c438) at common/Cond.h:56
#2 Pipe::DelayedDelivery::entry (this=0x7fbb1137c360) at msg/simple/Pipe.cc:203
#3 0x00007fbb03e1cdc5 in start_thread () from /lib64/libpthread.so.0
#4 0x00007fbb024a828d in ?? () from /lib64/libc.so.6
#5 0x0000000000000000 in ?? ()

Thread 79 (Thread 0x7fbb01131700 (LWP 16928)):
#0 0x00007fbb03e206d5 in pthread_cond_wait@@GLIBC_2.3.2 () from /lib64/libpthread.so.0
#1 0x00007fbb0604b74b in ceph::log::Log::entry (this=0x7fbb10f49800) at log/Log.cc:457
#2 0x00007fbb03e1cdc5 in start_thread () from /lib64/libpthread.so.0
#3 0x00007fbb024a828d in ?? () from /lib64/libc.so.6
#4 0x0000000000000000 in ?? ()

---Type <return> to continue, or q <return> to quit---
Thread 78 (Thread 0x7fbae4485700 (LWP 17025)):
#0 0x00007fbb03e20a82 in pthread_cond_timedwait@@GLIBC_2.3.2 () from /lib64/libpthread.so.0
#1 0x00007fbb060f43da in WaitUntil (when=..., mutex=..., this=0x7fbb1114ac58) at common/Cond.h:72
#2 WaitInterval (interval=..., mutex=..., cct=<optimized out>, this=0x7fbb1114ac58) at common/Cond.h:81
#3 ThreadPool::worker (this=0x7fbb1114abd8, wt=0x7fbb11192910) at common/WorkQueue.cc:151
#4 0x00007fbb060f5670 in ThreadPool::WorkThread::entry (this=<optimized out>) at common/WorkQueue.h:445
#5 0x00007fbb03e1cdc5 in start_thread () from /lib64/libpthread.so.0
#6 0x00007fbb024a828d in ?? () from /lib64/libc.so.6
#7 0x0000000000000000 in ?? ()

Thread 77 (Thread 0x7fbad7a55700 (LWP 22006)):
#0 0x00007fbb03e206d5 in pthread_cond_wait@@GLIBC_2.3.2 () from /lib64/libpthread.so.0
#1 0x00007fbb061ff0ad in Wait (mutex=..., this=0x7fbb11297248) at common/Cond.h:56
#2 Pipe::reader (this=0x7fbb11296000) at msg/simple/Pipe.cc:1529
#3 0x00007fbb062084bd in Pipe::Reader::entry (this=<optimized out>) at msg/simple/Pipe.h:52
#4 0x00007fbb03e1cdc5 in start_thread () from /lib64/libpthread.so.0
#5 0x00007fbb024a828d in ?? () from /lib64/libc.so.6
#6 0x0000000000000000 in ?? ()

Thread 76 (Thread 0x7fbadd16d700 (LWP 17308)):
#0 0x00007fbb024a8863 in open_by_handle_at () from /lib64/libc.so.6
#1 0x0000000000000000 in ?? ()

Thread 75 (Thread 0x7fbad7954700 (LWP 22008)):
#0 0x00007fbb03e206d5 in pthread_cond_wait@@GLIBC_2.3.2 () from /lib64/libpthread.so.0
#1 0x00007fbb061f60b9 in Wait (mutex=..., this=0x7fbb112fcaf8) at common/Cond.h:56
#2 Pipe::DelayedDelivery::entry (this=0x7fbb112fca20) at msg/simple/Pipe.cc:203
#3 0x00007fbb03e1cdc5 in start_thread () from /lib64/libpthread.so.0
#4 0x00007fbb024a828d in ?? () from /lib64/libc.so.6
#5 0x0000000000000000 in ?? ()

Thread 74 (Thread 0x7fbae4c86700 (LWP 17024)):
#0 0x00007fbb03e20a82 in pthread_cond_timedwait@@GLIBC_2.3.2 () from /lib64/libpthread.so.0
#1 0x00007fbb060f43da in WaitUntil (when=..., mutex=..., this=0x7fbb1114aac0) at common/Cond.h:72
#2 WaitInterval (interval=..., mutex=..., cct=<optimized out>, this=0x7fbb1114aac0) at common/Cond.h:81
#3 ThreadPool::worker (this=0x7fbb1114aa40, wt=0x7fbb111928b0) at common/WorkQueue.cc:151
#4 0x00007fbb060f5670 in ThreadPool::WorkThread::entry (this=<optimized out>) at common/WorkQueue.h:445
#5 0x00007fbb03e1cdc5 in start_thread () from /lib64/libpthread.so.0
#6 0x00007fbb024a828d in ?? () from /lib64/libc.so.6
#7 0x0000000000000000 in ?? ()

Thread 73 (Thread 0x7fbb056cc800 (LWP 16927)):
#0 0x00007fbb03e1def7 in pthread_join () from /lib64/libpthread.so.0
#1 0x00007fbb060e6310 in Thread::join (this=this@entry=0x7fbb10fa4440, prval=prval@entry=0x0) at common/Thread.cc:171
#2 0x00007fbb061c3f52 in DispatchQueue::wait (this=this@entry=0x7fbb10fa41d0) at msg/simple/DispatchQueue.cc:226
#3 0x00007fbb060dab5b in SimpleMessenger::wait (this=0x7fbb10fa4000) at msg/simple/SimpleMessenger.cc:543
#4 0x00007fbb059f60f6 in main (argc=<optimized out>, argv=<optimized out>) at ceph_osd.cc:623

Thread 72 (Thread 0x7fbad7053700 (LWP 21307)):
#0 0x00007fbb03e22f4d in __lll_lock_wait () from /lib64/libpthread.so.0
#1 0x00007fbb03e1ed02 in _L_lock_791 () from /lib64/libpthread.so.0
#2 0x00007fbb03e1ec08 in pthread_mutex_lock () from /lib64/libpthread.so.0
#3 0x00007fbb060b47d8 in Mutex::Lock (this=this@entry=0x7fbb13d0d170, no_lockdep=no_lockdep@entry=false) at common/Mutex.cc:110
#4 0x00007fbb061ff791 in Pipe::reader (this=0x7fbb13d0c000) at msg/simple/Pipe.cc:1606
#5 0x00007fbb062084bd in Pipe::Reader::entry (this=<optimized out>) at msg/simple/Pipe.h:52
#6 0x00007fbb03e1cdc5 in start_thread () from /lib64/libpthread.so.0
#7 0x00007fbb024a828d in ?? () from /lib64/libc.so.6
#8 0x0000000000000000 in ?? ()

Thread 71 (Thread 0x7fbae7c8c700 (LWP 17018)):
#0 0x00007fbb03e22f4d in __lll_lock_wait () from /lib64/libpthread.so.0
#1 0x00007fbb03e1ed02 in _L_lock_791 () from /lib64/libpthread.so.0
#2 0x00007fbb03e1ec08 in pthread_mutex_lock () from /lib64/libpthread.so.0
#3 0x00007fbb060b47d8 in Mutex::Lock (this=this@entry=0x7fbb13d0d170, no_lockdep=no_lockdep@entry=false) at common/Mutex.cc:110
#4 0x00007fbb060d85d6 in Locker (m=..., this=<synthetic pointer>) at common/Mutex.h:115
#5 is_connected (this=0x7fbb13d0c000) at msg/simple/Pipe.h:143
#6 SimpleMessenger::is_connected (this=<optimized out>, con=<optimized out>) at msg/simple/SimpleMessenger.cc:282

#7 0x00007fbb05a4dc06 in OSD::op_is_discardable (op=op@entry=0x7fbb12215b80) at osd/OSD.cc:8616
#8 0x00007fbb05b22cec in PG::can_discard_op (this=0x7fbb1142d000, op=std::shared_ptr (count 3, weak 0) 0x7fbb1144fa00) at osd/PG.cc:5353
#9 0x00007fbb05b23405 in PG::can_discard_request (this=this@entry=0x7fbb1142d000, op=std::shared_ptr (count 3, weak 0) 0x7fbb1144fa00) at osd/PG.cc:5439
#10 0x00007fbb05bc21e9 in ReplicatedPG::do_request (this=0x7fbb1142d000, op=std::shared_ptr (count 3, weak 0) 0x7fbb1144fa00, handle=...) at osd/ReplicatedPG.cc:1448
#11 0x00007fbb05a7607d in OSD::dequeue_op (this=0x7fbb1114a000, pg=..., op=std::shared_ptr (count 3, weak 0) 0x7fbb1144fa00, handle=...) at osd/OSD.cc:8815
#12 0x00007fbb05a762cd in PGQueueable::RunVis::operator() (this=this@entry=0x7fbae7c8a7b0, op=std::shared_ptr (count 3, weak 0) 0x7fbb1144fa00) at osd/OSD.cc:163
#13 0x00007fbb05a7adf9 in internal_visit<std::shared_ptr<OpRequest> > (operand=std::shared_ptr (count 3, weak 0) 0x7fbb1144fa00, this=<synthetic pointer>) at /usr/include/boost/variant/variant.hpp:1017
#14 visitation_impl_invoke_impl<boost::detail::variant::invoke_visitor<PGQueueable::RunVis>, void*, std::shared_ptr<OpRequest> > (storage=0x7fbae7c8a8d0, visitor=<synthetic pointer>) at /usr/include/boost/variant/detail/visitation_impl.hpp:130
#15 visitation_impl_invoke<boost::detail::variant::invoke_visitor<PGQueueable::RunVis>, void*, std::shared_ptr<OpRequest>, boost::variant<std::shared_ptr<OpRequest>, PGSnapTrim, PGScrub>::has_fallback_type_> (internal_which=<optimized out>, t=0x0, storage=0x7fbae7c8a8d0, visitor=<synthetic pointer>)
at /usr/include/boost/variant/detail/visitation_impl.hpp:173
#16 visitation_impl<mpl_::int_<0>, boost::detail::variant::visitation_impl_step<boost::mpl::l_iter<boost::mpl::l_item<mpl_::long_<3l>, std::shared_ptr<OpRequest>, boost::mpl::l_item<mpl_::long_<2l>, PGSnapTrim, boost::mpl::l_item<mpl_::long_<1l>, PGScrub, boost::mpl::l_end> > > >, boost::mpl::l_iter<boost::mpl::l_end> >, boost::detail::variant::invoke_visitor<PGQueueable::RunVis>, void*, boost::variant<std::shared_ptr<OpRequest>, PGSnapTrim, PGScrub>::has_fallback_type_> (no_backup_flag=..., storage=0x7fbae7c8a8d0, visitor=<synthetic pointer>, logical_which=<optimized out>, internal_which=<optimized out>)
at /usr/include/boost/variant/detail/visitation_impl.hpp:256
#17 internal_apply_visitor_impl<boost::detail::variant::invoke_visitor<PGQueueable::RunVis>, void*> (storage=0x7fbae7c8a8d0, visitor=<synthetic pointer>, logical_which=<optimized out>, internal_which=<optimized out>) at /usr/include/boost/variant/variant.hpp:2326
#18 internal_apply_visitor<boost::detail::variant::invoke_visitor<PGQueueable::RunVis> > (visitor=<synthetic pointer>, this=0x7fbae7c8a8c8) at /usr/include/boost/variant/variant.hpp:2337
#19 apply_visitor<PGQueueable::RunVis> (visitor=..., this=0x7fbae7c8a8c8) at /usr/include/boost/variant/variant.hpp:2360
#20 apply_visitor<PGQueueable::RunVis, boost::variant<std::shared_ptr<OpRequest>, PGSnapTrim, PGScrub> > (visitable=..., visitor=...) at /usr/include/boost/variant/detail/apply_visitor_unary.hpp:60
#21 run (handle=..., pg=..., osd=<optimized out>, this=0x7fbae7c8a8c8) at osd/OSD.h:392
#22 OSD::ShardedOpWQ::_process (this=0x7fbb1114b1e8, thread_index=<optimized out>, hb=<optimized out>) at osd/OSD.cc:8696
---Type <return> to continue, or q <return> to quit---
#23 0x00007fbb060f36e7 in ShardedThreadPool::shardedthreadpool_worker (this=0x7fbb1114a768, thread_index=<optimized out>) at common/WorkQueue.cc:340
#24 0x00007fbb060f5650 in ShardedThreadPool::WorkThreadSharded::entry (this=<optimized out>) at common/WorkQueue.h:684
#25 0x00007fbb03e1cdc5 in start_thread () from /lib64/libpthread.so.0
#26 0x00007fbb024a828d in ?? () from /lib64/libc.so.6
#27 0x0000000000000000 in ?? ()

Thread 70 (Thread 0x7fbade371700 (LWP 21176)):
#0 0x00007fbb03e206d5 in pthread_cond_wait@@GLIBC_2.3.2 () from /lib64/libpthread.so.0
#1 0x00007fbb061f60b9 in Wait (mutex=..., this=0x7fbb112fd518) at common/Cond.h:56
#2 Pipe::DelayedDelivery::entry (this=0x7fbb112fd440) at msg/simple/Pipe.cc:203
#3 0x00007fbb03e1cdc5 in start_thread () from /lib64/libpthread.so.0
#4 0x00007fbb024a828d in ?? () from /lib64/libc.so.6
#5 0x0000000000000000 in ?? ()

Thread 69 (Thread 0x7fbae5487700 (LWP 17023)):
#0 0x00007fbb03e20a82 in pthread_cond_timedwait@@GLIBC_2.3.2 () from /lib64/libpthread.so.0
#1 0x00007fbb060f43da in WaitUntil (when=..., mutex=..., this=0x7fbb1114a928) at common/Cond.h:72
#2 WaitInterval (interval=..., mutex=..., cct=<optimized out>, this=0x7fbb1114a928) at common/Cond.h:81
#3 ThreadPool::worker (this=0x7fbb1114a8a8, wt=0x7fbb111927f0) at common/WorkQueue.cc:151
#4 0x00007fbb060f5670 in ThreadPool::WorkThread::entry (this=<optimized out>) at common/WorkQueue.h:445
#5 0x00007fbb03e1cdc5 in start_thread () from /lib64/libpthread.so.0
#6 0x00007fbb024a828d in ?? () from /lib64/libc.so.6
#7 0x0000000000000000 in ?? ()

Thread 68 (Thread 0x7fbaed497700 (LWP 17007)):
#0 0x00007fbb03e206d5 in pthread_cond_wait@@GLIBC_2.3.2 () from /lib64/libpthread.so.0
#1 0x00007fbb061c535f in Wait (mutex=..., this=0x7fbb10fa51e8) at common/Cond.h:56
#2 DispatchQueue::run_local_delivery (this=0x7fbb10fa4fd0) at msg/simple/DispatchQueue.cc:114
#3 0x00007fbb060def5d in DispatchQueue::LocalDeliveryThread::entry (this=<optimized out>) at msg/simple/DispatchQueue.h:117
#4 0x00007fbb03e1cdc5 in start_thread () from /lib64/libpthread.so.0
#5 0x00007fbb024a828d in ?? () from /lib64/libc.so.6
#6 0x0000000000000000 in ?? ()

Thread 67 (Thread 0x7fbade674700 (LWP 19778)):
#0 0x00007fbb03e206d5 in pthread_cond_wait@@GLIBC_2.3.2 () from /lib64/libpthread.so.0
#1 0x00007fbb062055cf in Wait (mutex=..., this=0x7fbb117bd248) at common/Cond.h:56
#2 Pipe::writer (this=0x7fbb117bc000) at msg/simple/Pipe.cc:1864
#3 0x00007fbb062084dd in Pipe::Writer::entry (this=<optimized out>) at msg/simple/Pipe.h:63
#4 0x00007fbb03e1cdc5 in start_thread () from /lib64/libpthread.so.0
#5 0x00007fbb024a828d in ?? () from /lib64/libc.so.6
#6 0x0000000000000000 in ?? ()

Thread 66 (Thread 0x7fbaddb70700 (LWP 22726)):
#0 0x00007fbb0249dc3d in __statvfs_getflags () from /lib64/libc.so.6
#1 0x0000000000000000 in ?? ()

Thread 65 (Thread 0x7fbae748b700 (LWP 17019)):
#0 0x00007fbb03e20a82 in pthread_cond_timedwait@@GLIBC_2.3.2 () from /lib64/libpthread.so.0
#1 0x00007fbb05a7a6e5 in WaitUntil (when=..., mutex=..., this=0x7fbb10edc7d8) at common/Cond.h:72
#2 WaitInterval (interval=..., mutex=..., cct=<optimized out>, this=0x7fbb10edc7d8) at common/Cond.h:81
#3 OSD::ShardedOpWQ::_process (this=0x7fbb1114b1e8, thread_index=<optimized out>, hb=0x7fbb11193660) at osd/OSD.cc:8644
#4 0x00007fbb060f36e7 in ShardedThreadPool::shardedthreadpool_worker (this=0x7fbb1114a768, thread_index=<optimized out>) at common/WorkQueue.cc:340
#5 0x00007fbb060f5650 in ShardedThreadPool::WorkThreadSharded::entry (this=<optimized out>) at common/WorkQueue.h:684
#6 0x00007fbb03e1cdc5 in start_thread () from /lib64/libpthread.so.0
#7 0x00007fbb024a828d in ?? () from /lib64/libc.so.6
#8 0x0000000000000000 in ?? ()

Thread 64 (Thread 0x7fbaf049d700 (LWP 17001)):
#0 0x00007fbb03e206d5 in pthread_cond_wait@@GLIBC_2.3.2 () from /lib64/libpthread.so.0
#1 0x00007fbb061c535f in Wait (mutex=..., this=0x7fbb10fa43e8) at common/Cond.h:56
#2 DispatchQueue::run_local_delivery (this=0x7fbb10fa41d0) at msg/simple/DispatchQueue.cc:114
#3 0x00007fbb060def5d in DispatchQueue::LocalDeliveryThread::entry (this=<optimized out>) at msg/simple/DispatchQueue.h:117
#4 0x00007fbb03e1cdc5 in start_thread () from /lib64/libpthread.so.0
#5 0x00007fbb024a828d in ?? () from /lib64/libc.so.6
#6 0x0000000000000000 in ?? ()

Thread 63 (Thread 0x7fbadbd67700 (LWP 17324)):
#0 0x00007fbb03e206d5 in pthread_cond_wait@@GLIBC_2.3.2 () from /lib64/libpthread.so.0
#1 0x00007fbb061f60b9 in Wait (mutex=..., this=0x7fbb1104b5b8) at common/Cond.h:56
#2 Pipe::DelayedDelivery::entry (this=0x7fbb1104b4e0) at msg/simple/Pipe.cc:203
#3 0x00007fbb03e1cdc5 in start_thread () from /lib64/libpthread.so.0
#4 0x00007fbb024a828d in ?? () from /lib64/libc.so.6
#5 0x0000000000000000 in ?? ()

Thread 62 (Thread 0x7fbad985e700 (LWP 21074)):
#0 0x00007fbb03e206d5 in pthread_cond_wait@@GLIBC_2.3.2 () from /lib64/libpthread.so.0
#1 0x00007fbb062055cf in Wait (mutex=..., this=0x7fbb11288648) at common/Cond.h:56
#2 Pipe::writer (this=0x7fbb11287400) at msg/simple/Pipe.cc:1864
#3 0x00007fbb062084dd in Pipe::Writer::entry (this=<optimized out>) at msg/simple/Pipe.h:63
#4 0x00007fbb03e1cdc5 in start_thread () from /lib64/libpthread.so.0
#5 0x00007fbb024a828d in ?? () from /lib64/libc.so.6
#6 0x0000000000000000 in ?? ()

Thread 61 (Thread 0x7fbadd96e700 (LWP 21309)):
#0 0x00007fbb03e22f4d in __lll_lock_wait () from /lib64/libpthread.so.0
#1 0x00007fbb03e1ed02 in _L_lock_791 () from /lib64/libpthread.so.0
---Type <return> to continue, or q <return> to quit---
#2 0x00007fbb03e1ec08 in pthread_mutex_lock () from /lib64/libpthread.so.0
#3 0x00007fbb060b47d8 in Mutex::Lock (this=this@entry=0x7fbb13d0d170, no_lockdep=no_lockdep@entry=false) at common/Mutex.cc:110
#4 0x00007fbb060d85d6 in Locker (m=..., this=<synthetic pointer>) at common/Mutex.h:115
#5 is_connected (this=0x7fbb13d0c000) at msg/simple/Pipe.h:143
#6 SimpleMessenger::is_connected (this=<optimized out>, con=<optimized out>) at msg/simple/SimpleMessenger.cc:282
#7 0x00007fbb05a4dc06 in OSD::op_is_discardable (op=op@entry=0x7fbb12214dc0) at osd/OSD.cc:8616
#8 0x00007fbb05a82e9a in OSD::handle_op (this=this@entry=0x7fbb1114a000, op=std::shared_ptr (count 2, weak 0) 0x7fbb1144f800, osdmap=warning: RTTI symbol not found for class 'std::_Sp_counted_deleter<OSDMap const*, SharedLRU<unsigned int, OSDMap const, std::less<unsigned int>, std::hash<unsigned int> >::Cleanup, std::allocator<int>, (__gnu_cxx::_Lock_policy)2>'
warning: RTTI symbol not found for class 'std::_Sp_counted_deleter<OSDMap const*, SharedLRU<unsigned int, OSDMap const, std::less<unsigned int>, std::hash<unsigned int> >::Cleanup, std::allocator<int>, (__gnu_cxx::_Lock_policy)2>'
std::shared_ptr (count 31, weak 1) 0x7fbb126fda80) at osd/OSD.cc:8466
#9 0x00007fbb05a840ce in OSD::dispatch_op_fast (this=this@entry=0x7fbb1114a000, op=std::shared_ptr (count 2, weak 0) 0x7fbb1144f800, osdmap=warning: RTTI symbol not found for class 'std::_Sp_counted_deleter<OSDMap const*, SharedLRU<unsigned int, OSDMap const, std::less<unsigned int>, std::hash<unsigned int> >::Cleanup, std::allocator<int>, (__gnu_cxx::_Lock_policy)2>'
warning: RTTI symbol not found for class 'std::_Sp_counted_deleter<OSDMap const*, SharedLRU<unsigned int, OSDMap const, std::less<unsigned int>, std::hash<unsigned int> >::Cleanup, std::allocator<int>, (__gnu_cxx::_Lock_policy)2>'
std::shared_ptr (count 31, weak 1) 0x7fbb126fda80) at osd/OSD.cc:6135
#10 0x00007fbb05a843e8 in OSD::dispatch_session_waiting (this=this@entry=0x7fbb1114a000, session=session@entry=0x7fbb1345d260, osdmap=warning: RTTI symbol not found for class 'std::_Sp_counted_deleter<OSDMap const*, SharedLRU<unsigned int, OSDMap const, std::less<unsigned int>, std::hash<unsigned int> >::Cleanup, std::allocator<int>, (__gnu_cxx::_Lock_policy)2>'
warning: RTTI symbol not found for class 'std::_Sp_counted_deleter<OSDMap const*, SharedLRU<unsigned int, OSDMap const, std::less<unsigned int>, std::hash<unsigned int> >::Cleanup, std::allocator<int>, (__gnu_cxx::_Lock_policy)2>'
std::shared_ptr (count 31, weak 1) 0x7fbb126fda80) at osd/OSD.cc:5771
#11 0x00007fbb05a84794 in OSD::ms_fast_dispatch (this=0x7fbb1114a000, m=<optimized out>) at osd/OSD.cc:5883
#12 0x00007fbb061c3cf6 in ms_fast_dispatch (m=0x7fbb12214dc0, this=0x7fbb10fa4000) at msg/Messenger.h:555
#13 DispatchQueue::fast_dispatch (this=0x7fbb10fa41d0, m=m@entry=0x7fbb12214dc0) at msg/simple/DispatchQueue.cc:71
#14 0x00007fbb061f6372 in Pipe::DelayedDelivery::entry (this=0x7fbb11733560) at msg/simple/Pipe.cc:226
#15 0x00007fbb03e1cdc5 in start_thread () from /lib64/libpthread.so.0
#16 0x00007fbb024a828d in ?? () from /lib64/libc.so.6
#17 0x0000000000000000 in ?? ()

Thread 60 (Thread 0x7fbae0b7d700 (LWP 17062)):
#0 0x00007fbb0249dc3d in __statvfs_getflags () from /lib64/libc.so.6
#1 0x0000000000000000 in ?? ()

Thread 59 (Thread 0x7fbae948f700 (LWP 17015)):
#0 0x00007fbb03e20a82 in pthread_cond_timedwait@@GLIBC_2.3.2 () from /lib64/libpthread.so.0
#1 0x00007fbb05a7a6e5 in WaitUntil (when=..., mutex=..., this=0x7fbb10edc918) at common/Cond.h:72
#2 WaitInterval (interval=..., mutex=..., cct=<optimized out>, this=0x7fbb10edc918) at common/Cond.h:81
#3 OSD::ShardedOpWQ::_process (this=0x7fbb1114b1e8, thread_index=<optimized out>, hb=0x7fbb111935a0) at osd/OSD.cc:8644
#4 0x00007fbb060f36e7 in ShardedThreadPool::shardedthreadpool_worker (this=0x7fbb1114a768, thread_index=<optimized out>) at common/WorkQueue.cc:340
#5 0x00007fbb060f5650 in ShardedThreadPool::WorkThreadSharded::entry (this=<optimized out>) at common/WorkQueue.h:684
#6 0x00007fbb03e1cdc5 in start_thread () from /lib64/libpthread.so.0
#7 0x00007fbb024a828d in ?? () from /lib64/libc.so.6
#8 0x0000000000000000 in ?? ()

Thread 58 (Thread 0x7fbaf663f700 (LWP 16984)):
#0 0x00007fbb03e206d5 in pthread_cond_wait@@GLIBC_2.3.2 () from /lib64/libpthread.so.0
#1 0x00007fbb06031378 in Wait (mutex=..., this=0x7fbb10f941c0) at common/Cond.h:56
#2 Finisher::finisher_thread_entry (this=0x7fbb10f94160) at common/Finisher.cc:96
#3 0x00007fbb03e1cdc5 in start_thread () from /lib64/libpthread.so.0
#4 0x00007fbb024a828d in ?? () from /lib64/libc.so.6
#5 0x0000000000000000 in ?? ()

Thread 57 (Thread 0x7fbaecc96700 (LWP 17008)):
#0 0x00007fbb0249dc3d in __statvfs_getflags () from /lib64/libc.so.6
#1 0x0000000000000000 in ?? ()

Thread 56 (Thread 0x7fbad8357700 (LWP 18880)):
#0 0x00007fbb03e206d5 in pthread_cond_wait@@GLIBC_2.3.2 () from /lib64/libpthread.so.0
#1 0x00007fbb061f60b9 in Wait (mutex=..., this=0x7fbb11471098) at common/Cond.h:56
#2 Pipe::DelayedDelivery::entry (this=0x7fbb11470fc0) at msg/simple/Pipe.cc:203
#3 0x00007fbb03e1cdc5 in start_thread () from /lib64/libpthread.so.0
#4 0x00007fbb024a828d in ?? () from /lib64/libc.so.6
#5 0x0000000000000000 in ?? ()

Thread 55 (Thread 0x7fbadad65700 (LWP 17341)):
#0 0x00007fbb03e206d5 in pthread_cond_wait@@GLIBC_2.3.2 () from /lib64/libpthread.so.0
#1 0x00007fbb062055cf in Wait (mutex=..., this=0x7fbb112e3248) at common/Cond.h:56
#2 Pipe::writer (this=0x7fbb112e2000) at msg/simple/Pipe.cc:1864
#3 0x00007fbb062084dd in Pipe::Writer::entry (this=<optimized out>) at msg/simple/Pipe.h:63
#4 0x00007fbb03e1cdc5 in start_thread () from /lib64/libpthread.so.0
#5 0x00007fbb024a828d in ?? () from /lib64/libc.so.6
#6 0x0000000000000000 in ?? ()

Thread 54 (Thread 0x7fbaf7641700 (LWP 16982)):
#0 0x00007fbb03e20a82 in pthread_cond_timedwait@@GLIBC_2.3.2 () from /lib64/libpthread.so.0
#1 0x00007fbb060f43da in WaitUntil (when=..., mutex=..., this=0x7fbb10f90b20) at common/Cond.h:72
#2 WaitInterval (interval=..., mutex=..., cct=<optimized out>, this=0x7fbb10f90b20) at common/Cond.h:81
#3 ThreadPool::worker (this=0x7fbb10f90aa0, wt=0x7fbb11156160) at common/WorkQueue.cc:151
#4 0x00007fbb060f5670 in ThreadPool::WorkThread::entry (this=<optimized out>) at common/WorkQueue.h:445
#5 0x00007fbb03e1cdc5 in start_thread () from /lib64/libpthread.so.0
#6 0x00007fbb024a828d in ?? () from /lib64/libc.so.6
#7 0x0000000000000000 in ?? ()

Thread 53 (Thread 0x7fbae9c90700 (LWP 17014)):
#0 0x00007fbb03e20a82 in pthread_cond_timedwait@@GLIBC_2.3.2 () from /lib64/libpthread.so.0
#1 0x00007fbb05a7a6e5 in WaitUntil (when=..., mutex=..., this=0x7fbb10edc7d8) at common/Cond.h:72
#2 WaitInterval (interval=..., mutex=..., cct=<optimized out>, this=0x7fbb10edc7d8) at common/Cond.h:81
#3 OSD::ShardedOpWQ::_process (this=0x7fbb1114b1e8, thread_index=<optimized out>, hb=0x7fbb11193540) at osd/OSD.cc:8644
#4 0x00007fbb060f36e7 in ShardedThreadPool::shardedthreadpool_worker (this=0x7fbb1114a768, thread_index=<optimized out>) at common/WorkQueue.cc:340
#5 0x00007fbb060f5650 in ShardedThreadPool::WorkThreadSharded::entry (this=<optimized out>) at common/WorkQueue.h:684
#6 0x00007fbb03e1cdc5 in start_thread () from /lib64/libpthread.so.0
#7 0x00007fbb024a828d in ?? () from /lib64/libc.so.6
#8 0x0000000000000000 in ?? ()

Thread 52 (Thread 0x7fbae037c700 (LWP 22007)):
#0 0x00007fbb03e206d5 in pthread_cond_wait@@GLIBC_2.3.2 () from /lib64/libpthread.so.0
#1 0x00007fbb062055cf in Wait (mutex=..., this=0x7fbb11297248) at common/Cond.h:56
#2 Pipe::writer (this=0x7fbb11296000) at msg/simple/Pipe.cc:1864
---Type <return> to continue, or q <return> to quit---
#3 0x00007fbb062084dd in Pipe::Writer::entry (this=<optimized out>) at msg/simple/Pipe.h:63
#4 0x00007fbb03e1cdc5 in start_thread () from /lib64/libpthread.so.0
#5 0x00007fbb024a828d in ?? () from /lib64/libc.so.6
#6 0x0000000000000000 in ?? ()

Thread 51 (Thread 0x7fbaee499700 (LWP 17005)):
#0 0x00007fbb0249dc3d in __statvfs_getflags () from /lib64/libc.so.6
#1 0x0000000000000000 in ?? ()

Thread 50 (Thread 0x7fbadb566700 (LWP 17336)):
#0 0x00007fbb03e206d5 in pthread_cond_wait@@GLIBC_2.3.2 () from /lib64/libpthread.so.0
#1 0x00007fbb061f60b9 in Wait (mutex=..., this=0x7fbb1104ba38) at common/Cond.h:56
#2 Pipe::DelayedDelivery::entry (this=0x7fbb1104b960) at msg/simple/Pipe.cc:203
#3 0x00007fbb03e1cdc5 in start_thread () from /lib64/libpthread.so.0
#4 0x00007fbb024a828d in ?? () from /lib64/libc.so.6
#5 0x0000000000000000 in ?? ()

Thread 49 (Thread 0x7fbae0c7e700 (LWP 17061)):
#0 0x00007fbb0249dc3d in __statvfs_getflags () from /lib64/libc.so.6
#1 0x0000000000000000 in ?? ()

Thread 48 (Thread 0x7fbade573700 (LWP 21073)):
#0 0x00007fbb03e206d5 in pthread_cond_wait@@GLIBC_2.3.2 () from /lib64/libpthread.so.0
#1 0x00007fbb061ff0ad in Wait (mutex=..., this=0x7fbb11288648) at common/Cond.h:56
#2 Pipe::reader (this=0x7fbb11287400) at msg/simple/Pipe.cc:1529
#3 0x00007fbb062084bd in Pipe::Reader::entry (this=<optimized out>) at msg/simple/Pipe.h:52
#4 0x00007fbb03e1cdc5 in start_thread () from /lib64/libpthread.so.0
#5 0x00007fbb024a828d in ?? () from /lib64/libc.so.6
#6 0x0000000000000000 in ?? ()

Thread 47 (Thread 0x7fbaeb493700 (LWP 17011)):
#0 0x00007fbb03e20a82 in pthread_cond_timedwait@@GLIBC_2.3.2 () from /lib64/libpthread.so.0
#1 0x00007fbb060f43da in WaitUntil (when=..., mutex=..., this=0x7fbb1114a650) at common/Cond.h:72
#2 WaitInterval (interval=..., mutex=..., cct=<optimized out>, this=0x7fbb1114a650) at common/Cond.h:81
#3 ThreadPool::worker (this=0x7fbb1114a5d0, wt=0x7fbb11192760) at common/WorkQueue.cc:151
#4 0x00007fbb060f5670 in ThreadPool::WorkThread::entry (this=<optimized out>) at common/WorkQueue.h:445
#5 0x00007fbb03e1cdc5 in start_thread () from /lib64/libpthread.so.0
#6 0x00007fbb024a828d in ?? () from /lib64/libc.so.6
#7 0x0000000000000000 in ?? ()

Thread 46 (Thread 0x7fbadda6f700 (LWP 21308)):
#0 0x00007fbb03e22f4d in __lll_lock_wait () from /lib64/libpthread.so.0
#1 0x00007fbb03e254bc in _L_cond_lock_792 () from /lib64/libpthread.so.0
#2 0x00007fbb03e25398 in __pthread_mutex_cond_lock () from /lib64/libpthread.so.0
#3 0x00007fbb03e20765 in pthread_cond_wait@@GLIBC_2.3.2 () from /lib64/libpthread.so.0
#4 0x00007fbb062055cf in Wait (mutex=..., this=0x7fbb13d0d248) at common/Cond.h:56
#5 Pipe::writer (this=0x7fbb13d0c000) at msg/simple/Pipe.cc:1864

#6 0x00007fbb062084dd in Pipe::Writer::entry (this=<optimized out>) at msg/simple/Pipe.h:63
#7 0x00007fbb03e1cdc5 in start_thread () from /lib64/libpthread.so.0
#8 0x00007fbb024a828d in ?? () from /lib64/libc.so.6
#9 0x0000000000000000 in ?? ()

Thread 45 (Thread 0x7fbadab63700 (LWP 21310)):
#0 0x00007fbb03e206d5 in pthread_cond_wait@@GLIBC_2.3.2 () from /lib64/libpthread.so.0
#1 0x00007fbb061ed78f in Wait (mutex=..., this=0x7fbb11733638) at common/Cond.h:56
#2 Pipe::DelayedDelivery::stop_fast_dispatching (this=0x7fbb11733560) at msg/simple/Pipe.cc:249
#3 0x00007fbb061ee99f in Pipe::stop_and_wait (this=this@entry=0x7fbb13d0c000) at msg/simple/Pipe.cc:1502
#4 0x00007fbb061fcd3f in Pipe::accept (this=this@entry=0x7fbb117be800) at msg/simple/Pipe.cc:696
#5 0x00007fbb062007cf in Pipe::reader (this=0x7fbb117be800) at msg/simple/Pipe.cc:1517
#6 0x00007fbb062084bd in Pipe::Reader::entry (this=<optimized out>) at msg/simple/Pipe.h:52
#7 0x00007fbb03e1cdc5 in start_thread () from /lib64/libpthread.so.0
#8 0x00007fbb024a828d in ?? () from /lib64/libc.so.6
#9 0x0000000000000000 in ?? ()

Thread 44 (Thread 0x7fbae1c80700 (LWP 17030)):
#0 0x00007fbb03e206d5 in pthread_cond_wait@@GLIBC_2.3.2 () from /lib64/libpthread.so.0
#1 0x00007fbb060ec6ff in Wait (mutex=..., this=0x7fbb1114c1b0) at common/Cond.h:56
#2 SafeTimer::timer_thread (this=0x7fbb1114c1a0) at common/Timer.cc:116
#3 0x00007fbb060edd3d in SafeTimerThread::entry (this=<optimized out>) at common/Timer.cc:38
#4 0x00007fbb03e1cdc5 in start_thread () from /lib64/libpthread.so.0
#5 0x00007fbb024a828d in ?? () from /lib64/libc.so.6
#6 0x0000000000000000 in ?? ()

Thread 43 (Thread 0x7fbaef49b700 (LWP 17003)):
#0 0x00007fbb03e22f4d in __lll_lock_wait () from /lib64/libpthread.so.0
#1 0x00007fbb03e1ed02 in _L_lock_791 () from /lib64/libpthread.so.0
#2 0x00007fbb03e1ec08 in pthread_mutex_lock () from /lib64/libpthread.so.0
#3 0x00007fbb060b47d8 in Mutex::Lock (this=this@entry=0x7ffd68175960, no_lockdep=no_lockdep@entry=false) at common/Mutex.cc:110
#4 0x00007fbb05a63e58 in Locker (m=..., this=<synthetic pointer>) at common/Mutex.h:115
















#5 sub_want_increment (flags=1, start=35,
what="\230vR\023\273\177\000\000\000\341\335\177\206\261\006\026ЍI\357\272\177\000\000\000\240\024\021\273\177\000\000\300\215I\357\272\177\000\000\030\263\024\021\273\177\000\000#\000\000\000\000\000\000\000ЍI\357\272\177\000\000\200%\367\022\273\177\000\000\254@\246\005\273\177\000\000#\000\000\000\000\000\000\000۪\004\006\273\177\000\000؍I\357\272\177\000\000\000\341\335\177\206\261\006\026\000\240\024\021\273\177\000\000@\225I\357\272\177\000\000\"\000\000\000\000\000\000\000\277A\246\005\273\177\000\000\200\212\031\021\273\177\000\000@\301\355\020\273\177\000\000@\301\355\020\273\177\000\000\312V\321\002\"\000\000\000\000(H\021\273\177\000\000\340u\330\022\273\177\000\000\370\216\367\002\273\177\000\000\060o\367\002"...<Address 0x7fbaf0ca0792 out of bounds>, this=0x7ffd681757d0) at mon/MonClient.h:280
#6 OSD::osdmap_subscribe (this=this@entry=0x7fbb1114a000, epoch=35, force_request=force_request@entry=false) at osd/OSD.cc:6535
#7 0x00007fbb05a640ac in OSD::wait_for_new_map (this=this@entry=0x7fbb1114a000, op=std::shared_ptr (count 4, weak 0) 0x7fbb11482800) at osd/OSD.cc:6468
#8 0x00007fbb05a641bf in OSD::require_same_or_newer_map (this=this@entry=0x7fbb1114a000, op=std::shared_ptr (count 4, weak 0) 0x7fbb11482800, epoch=35, is_fast_dispatch=is_fast_dispatch@entry=false) at osd/OSD.cc:7370
---Type <return> to continue, or q <return> to quit---
#9 0x00007fbb05a9a19c in OSD::handle_pg_notify (this=0x7fbb1114a000, op=std::shared_ptr (count 4, weak 0) 0x7fbb11482800) at osd/OSD.cc:7766
#10 0x00007fbb05a9b730 in OSD::dispatch_op (this=this@entry=0x7fbb1114a000, op=std::shared_ptr (count 4, weak 0) 0x7fbb11482800) at osd/OSD.cc:6078
#11 0x00007fbb05a9c35e in OSD::_dispatch (this=this@entry=0x7fbb1114a000, m=m@entry=0x7fbb12f72580) at osd/OSD.cc:6252
#12 0x00007fbb05a9ca1f in OSD::ms_dispatch (this=0x7fbb1114a000, m=0x7fbb12f72580) at osd/OSD.cc:5755
#13 0x00007fbb061c5b9a in ms_deliver_dispatch (m=0x7fbb12f72580, this=0x7fbb10fa4700) at msg/Messenger.h:584
#14 DispatchQueue::entry (this=0x7fbb10fa48d0) at msg/simple/DispatchQueue.cc:185
#15 0x00007fbb060def3d in DispatchQueue::DispatchThread::entry (this=<optimized out>) at msg/simple/DispatchQueue.h:103
#16 0x00007fbb03e1cdc5 in start_thread () from /lib64/libpthread.so.0
#17 0x00007fbb024a828d in ?? () from /lib64/libc.so.6
#18 0x0000000000000000 in ?? ()

Thread 42 (Thread 0x7fbadef76700 (LWP 17269)):
#0 0x00007fbb03e22f4d in __lll_lock_wait () from /lib64/libpthread.so.0
#1 0x00007fbb03e1ed02 in _L_lock_791 () from /lib64/libpthread.so.0
#2 0x00007fbb03e1ec08 in pthread_mutex_lock () from /lib64/libpthread.so.0
#3 0x00007fbb060b47d8 in Mutex::Lock (this=this@entry=0x7ffd68175960, no_lockdep=no_lockdep@entry=false) at common/Mutex.cc:110
#4 0x00007fbb05a63e58 in Locker (m=..., this=<synthetic pointer>) at common/Mutex.h:115
#5 sub_want_increment (flags=1, start=35,
what="\230~\330\022\273\177\000\000\000\341\335\177\206\261\006\026\000\240\024\021\273\177\000\000@\301\355\020\273\177\000\000@Z\376\021\273\177\000\000\360C\367\336\272\177\000\000\000\240\024\021\273\177\000\000\060\256\024\021\273\177\000\000\240Fk\021\273\177\000\000\352s\247\005\273\177\000\000\200\332o\022\273\177\000\000\000\240\355\022\273\177\000\000\220D\367\336\272\177\000\000\060\000\000\000\273\177\000\000\314C\367\336\272\177\000\000\200|/\022\273\177\000\000P\200#\021\273\177\000\000\334Ӵ\021\273\177\000\000X\200#\021\273\177\000\000H\200#\021\273\177\000\000@\200#\021\273\177\000\000\312V\321\002\002", '\000' <repeats 15 times>, "\002\000\000\000\000\204.\021\273\177\000\000\242U\321\002"...<Address 0x7fbadf97a000 out of bounds>, this=0x7ffd681757d0) at mon/MonClient.h:280
#6 OSD::osdmap_subscribe (this=this@entry=0x7fbb1114a000, epoch=35, force_request=force_request@entry=false) at osd/OSD.cc:6535
#7 0x00007fbb05a773ea in OSD::handle_osd_ping (this=this@entry=0x7fbb1114a000, m=m@entry=0x7fbb12eda000) at osd/OSD.cc:4001
#8 0x00007fbb05a7866b in OSD::heartbeat_dispatch (this=0x7fbb1114a000, m=0x7fbb12eda000) at osd/OSD.cc:5713
#9 0x00007fbb06213d9b in ms_fast_dispatch (m=<optimized out>, this=0x7fbb10fac000) at msg/Messenger.h:555
#10 AsyncConnection::DelayedDelivery::do_request (this=<optimized out>, id=<optimized out>) at msg/async/AsyncConnection.cc:2453
#11 0x00007fbb061c8626 in EventCenter::process_time_events (this=this@entry=0x7fbb10faa848) at msg/async/Event.cc:307
#12 0x00007fbb061c9d78 in EventCenter::process_events (this=this@entry=0x7fbb10faa848, timeout_microseconds=<optimized out>, timeout_microseconds@entry=30000000) at msg/async/Event.cc:392
#13 0x00007fbb061a90e0 in Worker::entry (this=0x7fbb10faa800) at msg/async/AsyncMessenger.cc:294
#14 0x00007fbb03e1cdc5 in start_thread () from /lib64/libpthread.so.0
#15 0x00007fbb024a828d in ?? () from /lib64/libc.so.6
#16 0x0000000000000000 in ?? ()

Thread 41 (Thread 0x7fbaf3e3a700 (LWP 16989)):
#0 0x00007fbb03e206d5 in pthread_cond_wait@@GLIBC_2.3.2 () from /lib64/libpthread.so.0
#1 0x00007fbb06031378 in Wait (mutex=..., this=0x7fbb10f948a0) at common/Cond.h:56
#2 Finisher::finisher_thread_entry (this=0x7fbb10f94840) at common/Finisher.cc:96
#3 0x00007fbb03e1cdc5 in start_thread () from /lib64/libpthread.so.0
#4 0x00007fbb024a828d in ?? () from /lib64/libc.so.6
#5 0x0000000000000000 in ?? ()

Thread 40 (Thread 0x7fbae147f700 (LWP 17031)):
#0 0x00007fbb03e206d5 in pthread_cond_wait@@GLIBC_2.3.2 () from /lib64/libpthread.so.0
#1 0x00007fbb05aac57e in Cond::Wait (this=0x7fbb1114c050, mutex=...) at common/Cond.h:56
#2 0x00007fbb05a4412c in OSDService::agent_entry (this=0x7fbb1114b860) at osd/OSD.cc:552
#3 0x00007fbb05ab242d in OSDService::AgentThread::entry (this=<optimized out>) at osd/OSD.h:676
#4 0x00007fbb03e1cdc5 in start_thread () from /lib64/libpthread.so.0
#5 0x00007fbb024a828d in ?? () from /lib64/libc.so.6
#6 0x0000000000000000 in ?? ()

Thread 39 (Thread 0x7fbadaa62700 (LWP 17353)):
#0 0x00007fbb03e206d5 in pthread_cond_wait@@GLIBC_2.3.2 () from /lib64/libpthread.so.0
#1 0x00007fbb061ff0ad in Wait (mutex=..., this=0x7fbb112e3248) at common/Cond.h:56
#2 Pipe::reader (this=0x7fbb112e2000) at msg/simple/Pipe.cc:1529
#3 0x00007fbb062084bd in Pipe::Reader::entry (this=<optimized out>) at msg/simple/Pipe.h:52
#4 0x00007fbb03e1cdc5 in start_thread () from /lib64/libpthread.so.0
#5 0x00007fbb024a828d in ?? () from /lib64/libc.so.6
#6 0x0000000000000000 in ?? ()

Thread 38 (Thread 0x7fbae6489700 (LWP 17021)):
#0 0x00007fbb03e20a82 in pthread_cond_timedwait@@GLIBC_2.3.2 () from /lib64/libpthread.so.0
#1 0x00007fbb05a7a6e5 in WaitUntil (when=..., mutex=..., this=0x7fbb10edca58) at common/Cond.h:72
#2 WaitInterval (interval=..., mutex=..., cct=<optimized out>, this=0x7fbb10edca58) at common/Cond.h:81
#3 OSD::ShardedOpWQ::_process (this=0x7fbb1114b1e8, thread_index=<optimized out>, hb=0x7fbb111936c0) at osd/OSD.cc:8644
#4 0x00007fbb060f36e7 in ShardedThreadPool::shardedthreadpool_worker (this=0x7fbb1114a768, thread_index=<optimized out>) at common/WorkQueue.cc:340
#5 0x00007fbb060f5650 in ShardedThreadPool::WorkThreadSharded::entry (this=<optimized out>) at common/WorkQueue.h:684
#6 0x00007fbb03e1cdc5 in start_thread () from /lib64/libpthread.so.0
#7 0x00007fbb024a828d in ?? () from /lib64/libc.so.6
#8 0x0000000000000000 in ?? ()

Thread 37 (Thread 0x7fbae2481700 (LWP 17029)):
#0 0x00007fbb03e206d5 in pthread_cond_wait@@GLIBC_2.3.2 () from /lib64/libpthread.so.0
#1 0x00007fbb060ec6ff in Wait (mutex=..., this=0x7fbb1114c460) at common/Cond.h:56
#2 SafeTimer::timer_thread (this=0x7fbb1114c450) at common/Timer.cc:116
#3 0x00007fbb060edd3d in SafeTimerThread::entry (this=<optimized out>) at common/Timer.cc:38
#4 0x00007fbb03e1cdc5 in start_thread () from /lib64/libpthread.so.0
#5 0x00007fbb024a828d in ?? () from /lib64/libc.so.6
#6 0x0000000000000000 in ?? ()

Thread 36 (Thread 0x7fbaf463b700 (LWP 16988)):
#0 0x00007fbb03e206d5 in pthread_cond_wait@@GLIBC_2.3.2 () from /lib64/libpthread.so.0
#1 0x00007fbb06031378 in Wait (mutex=..., this=0x7fbb10f94740) at common/Cond.h:56
#2 Finisher::finisher_thread_entry (this=0x7fbb10f946e0) at common/Finisher.cc:96
#3 0x00007fbb03e1cdc5 in start_thread () from /lib64/libpthread.so.0
#4 0x00007fbb024a828d in ?? () from /lib64/libc.so.6
#5 0x0000000000000000 in ?? ()
---Type <return> to continue, or q <return> to quit---

Thread 35 (Thread 0x7fbae5c88700 (LWP 17022)):
#0 0x00007fbb03e20a82 in pthread_cond_timedwait@@GLIBC_2.3.2 () from /lib64/libpthread.so.0
#1 0x00007fbb05a7a6e5 in WaitUntil (when=..., mutex=..., this=0x7fbb10edcb98) at common/Cond.h:72
#2 WaitInterval (interval=..., mutex=..., cct=<optimized out>, this=0x7fbb10edcb98) at common/Cond.h:81
#3 OSD::ShardedOpWQ::_process (this=0x7fbb1114b1e8, thread_index=<optimized out>, hb=0x7fbb111936f0) at osd/OSD.cc:8644
#4 0x00007fbb060f36e7 in ShardedThreadPool::shardedthreadpool_worker (this=0x7fbb1114a768, thread_index=<optimized out>) at common/WorkQueue.cc:340
#5 0x00007fbb060f5650 in ShardedThreadPool::WorkThreadSharded::entry (this=<optimized out>) at common/WorkQueue.h:684
#6 0x00007fbb03e1cdc5 in start_thread () from /lib64/libpthread.so.0
#7 0x00007fbb024a828d in ?? () from /lib64/libc.so.6
#8 0x0000000000000000 in ?? ()

Thread 34 (Thread 0x7fbaf6e40700 (LWP 16983)):
#0 0x00007fbb03e20a82 in pthread_cond_timedwait@@GLIBC_2.3.2 () from /lib64/libpthread.so.0
#1 0x00007fbb060f43da in WaitUntil (when=..., mutex=..., this=0x7fbb10f90b20) at common/Cond.h:72
#2 WaitInterval (interval=..., mutex=..., cct=<optimized out>, this=0x7fbb10f90b20) at common/Cond.h:81
#3 ThreadPool::worker (this=0x7fbb10f90aa0, wt=0x7fbb111561c0) at common/WorkQueue.cc:151
#4 0x00007fbb060f5670 in ThreadPool::WorkThread::entry (this=<optimized out>) at common/WorkQueue.h:445
#5 0x00007fbb03e1cdc5 in start_thread () from /lib64/libpthread.so.0
#6 0x00007fbb024a828d in ?? () from /lib64/libc.so.6
#7 0x0000000000000000 in ?? ()

Thread 33 (Thread 0x7fbae2c82700 (LWP 17028)):
#0 0x00007fbb03e206d5 in pthread_cond_wait@@GLIBC_2.3.2 () from /lib64/libpthread.so.0
#1 0x00007fbb06031378 in Wait (mutex=..., this=0x7fbb1114c308) at common/Cond.h:56
#2 Finisher::finisher_thread_entry (this=0x7fbb1114c2a8) at common/Finisher.cc:96
#3 0x00007fbb03e1cdc5 in start_thread () from /lib64/libpthread.so.0
#4 0x00007fbb024a828d in ?? () from /lib64/libc.so.6
#5 0x0000000000000000 in ?? ()

Thread 32 (Thread 0x7fbae3483700 (LWP 17027)):
#0 0x00007fbb03e206d5 in pthread_cond_wait@@GLIBC_2.3.2 () from /lib64/libpthread.so.0
#1 0x00007fbb06031378 in Wait (mutex=..., this=0x7fbb1114c6a8) at common/Cond.h:56
#2 Finisher::finisher_thread_entry (this=0x7fbb1114c648) at common/Finisher.cc:96
#3 0x00007fbb03e1cdc5 in start_thread () from /lib64/libpthread.so.0
#4 0x00007fbb024a828d in ?? () from /lib64/libc.so.6
#5 0x0000000000000000 in ?? ()

Thread 31 (Thread 0x7fbae848d700 (LWP 17017)):
#0 0x00007fbb03e20a82 in pthread_cond_timedwait@@GLIBC_2.3.2 () from /lib64/libpthread.so.0
#1 0x00007fbb05a7a6e5 in WaitUntil (when=..., mutex=..., this=0x7fbb10edcb98) at common/Cond.h:72
#2 WaitInterval (interval=..., mutex=..., cct=<optimized out>, this=0x7fbb10edcb98) at common/Cond.h:81
#3 OSD::ShardedOpWQ::_process (this=0x7fbb1114b1e8, thread_index=<optimized out>, hb=0x7fbb11193600) at osd/OSD.cc:8644
#4 0x00007fbb060f36e7 in ShardedThreadPool::shardedthreadpool_worker (this=0x7fbb1114a768, thread_index=<optimized out>) at common/WorkQueue.cc:340
#5 0x00007fbb060f5650 in ShardedThreadPool::WorkThreadSharded::entry (this=<optimized out>) at common/WorkQueue.h:684
#6 0x00007fbb03e1cdc5 in start_thread () from /lib64/libpthread.so.0
#7 0x00007fbb024a828d in ?? () from /lib64/libc.so.6
#8 0x0000000000000000 in ?? ()

Thread 30 (Thread 0x7fbaf4e3c700 (LWP 16987)):
#0 0x00007fbb03e206d5 in pthread_cond_wait@@GLIBC_2.3.2 () from /lib64/libpthread.so.0
#1 0x00007fbb06031378 in Wait (mutex=..., this=0x7fbb10f945e0) at common/Cond.h:56
#2 Finisher::finisher_thread_entry (this=0x7fbb10f94580) at common/Finisher.cc:96
#3 0x00007fbb03e1cdc5 in start_thread () from /lib64/libpthread.so.0
#4 0x00007fbb024a828d in ?? () from /lib64/libc.so.6
#5 0x0000000000000000 in ?? ()

Thread 29 (Thread 0x7fbae6c8a700 (LWP 17020)):
#0 0x00007fbb03e20a82 in pthread_cond_timedwait@@GLIBC_2.3.2 () from /lib64/libpthread.so.0
#1 0x00007fbb05a7a6e5 in WaitUntil (when=..., mutex=..., this=0x7fbb10edc918) at common/Cond.h:72
#2 WaitInterval (interval=..., mutex=..., cct=<optimized out>, this=0x7fbb10edc918) at common/Cond.h:81
#3 OSD::ShardedOpWQ::_process (this=0x7fbb1114b1e8, thread_index=<optimized out>, hb=0x7fbb11193690) at osd/OSD.cc:8644
#4 0x00007fbb060f36e7 in ShardedThreadPool::shardedthreadpool_worker (this=0x7fbb1114a768, thread_index=<optimized out>) at common/WorkQueue.cc:340
#5 0x00007fbb060f5650 in ShardedThreadPool::WorkThreadSharded::entry (this=<optimized out>) at common/WorkQueue.h:684
#6 0x00007fbb03e1cdc5 in start_thread () from /lib64/libpthread.so.0
#7 0x00007fbb024a828d in ?? () from /lib64/libc.so.6
#8 0x0000000000000000 in ?? ()

Thread 28 (Thread 0x7fbaea491700 (LWP 17013)):
#0 0x00007fbb03e22f4d in __lll_lock_wait () from /lib64/libpthread.so.0
#1 0x00007fbb03e1ed02 in _L_lock_791 () from /lib64/libpthread.so.0
#2 0x00007fbb03e1ec08 in pthread_mutex_lock () from /lib64/libpthread.so.0
#3 0x00007fbb060b47d8 in Mutex::Lock (this=this@entry=0x7fbb1142d528, no_lockdep=no_lockdep@entry=false) at common/Mutex.cc:110
#4 0x00007fbb05b0df83 in PG::lock (this=this@entry=0x7fbb1142d000, no_lockdep=no_lockdep@entry=false) at osd/PG.cc:265
#5 0x00007fbb05b0e11e in PG::lock_suspend_timeout (this=0x7fbb1142d000, handle=...) at osd/PG.cc:259
#6 0x00007fbb05a7a869 in OSD::ShardedOpWQ::_process (this=0x7fbb1114b1e8, thread_index=<optimized out>, hb=0x7fbb11193510) at osd/OSD.cc:8658
#7 0x00007fbb060f36e7 in ShardedThreadPool::shardedthreadpool_worker (this=0x7fbb1114a768, thread_index=<optimized out>) at common/WorkQueue.cc:340
#8 0x00007fbb060f5650 in ShardedThreadPool::WorkThreadSharded::entry (this=<optimized out>) at common/WorkQueue.h:684
#9 0x00007fbb03e1cdc5 in start_thread () from /lib64/libpthread.so.0
#10 0x00007fbb024a828d in ?? () from /lib64/libc.so.6
#11 0x0000000000000000 in ?? ()

Thread 27 (Thread 0x7fbaf7e42700 (LWP 16981)):
#0 0x00007fbb03e206d5 in pthread_cond_wait@@GLIBC_2.3.2 () from /lib64/libpthread.so.0
#1 0x00007fbb06031378 in Wait (mutex=..., this=0x7fbb10f90080) at common/Cond.h:56
#2 Finisher::finisher_thread_entry (this=0x7fbb10f90020) at common/Finisher.cc:96
---Type <return> to continue, or q <return> to quit---
#3 0x00007fbb03e1cdc5 in start_thread () from /lib64/libpthread.so.0
#4 0x00007fbb024a828d in ?? () from /lib64/libc.so.6
#5 0x0000000000000000 in ?? ()

Thread 26 (Thread 0x7fbaebc94700 (LWP 17010)):
#0 0x00007fbb03e206d5 in pthread_cond_wait@@GLIBC_2.3.2 () from /lib64/libpthread.so.0
#1 0x00007fbb06031378 in Wait (mutex=..., this=0x7ffd68175ae0) at common/Cond.h:56
#2 Finisher::finisher_thread_entry (this=0x7ffd68175a80) at common/Finisher.cc:96
#3 0x00007fbb03e1cdc5 in start_thread () from /lib64/libpthread.so.0
#4 0x00007fbb024a828d in ?? () from /lib64/libc.so.6
#5 0x0000000000000000 in ?? ()

Thread 25 (Thread 0x7fbae3c84700 (LWP 17026)):
#0 0x00007fbb03e22f4d in __lll_lock_wait () from /lib64/libpthread.so.0
#1 0x00007fbb03e254bc in _L_cond_lock_792 () from /lib64/libpthread.so.0
#2 0x00007fbb03e25398 in __pthread_mutex_cond_lock () from /lib64/libpthread.so.0
#3 0x00007fbb03e20b54 in pthread_cond_timedwait@@GLIBC_2.3.2 () from /lib64/libpthread.so.0
#4 0x00007fbb05a6df4d in WaitUntil (when=..., mutex=..., this=0x7fbb1114aeb8) at common/Cond.h:72
#5 WaitInterval (interval=..., mutex=..., cct=<optimized out>, this=0x7fbb1114aeb8) at common/Cond.h:81
#6 OSD::heartbeat_entry (this=0x7fbb1114a000) at osd/OSD.cc:4021
#7 0x00007fbb05ad941d in OSD::T_Heartbeat::entry (this=<optimized out>) at osd/OSD.h:1590
#8 0x00007fbb03e1cdc5 in start_thread () from /lib64/libpthread.so.0
#9 0x00007fbb024a828d in ?? () from /lib64/libc.so.6
#10 0x0000000000000000 in ?? ()

Thread 24 (Thread 0x7fbaedc98700 (LWP 17006)):
#0 0x00007fbb03e206d5 in pthread_cond_wait@@GLIBC_2.3.2 () from /lib64/libpthread.so.0
#1 0x00007fbb061c56db in Wait (mutex=..., this=0x7fbb10fa5038) at common/Cond.h:56
#2 DispatchQueue::entry (this=0x7fbb10fa4fd0) at msg/simple/DispatchQueue.cc:196
#3 0x00007fbb060def3d in DispatchQueue::DispatchThread::entry (this=<optimized out>) at msg/simple/DispatchQueue.h:103
#4 0x00007fbb03e1cdc5 in start_thread () from /lib64/libpthread.so.0
#5 0x00007fbb024a828d in ?? () from /lib64/libc.so.6
#6 0x0000000000000000 in ?? ()

Thread 23 (Thread 0x7fbaf9645700 (LWP 16957)):
#0 0x00007fbb03e206d5 in pthread_cond_wait@@GLIBC_2.3.2 () from /lib64/libpthread.so.0
#1 0x00007fbb05e065af in Wait (mutex=..., this=0x7fbb10f90820) at common/Cond.h:56
#2 WBThrottle::get_next_should_flush (this=this@entry=0x7fbb10f906f8, next=next@entry=0x7fbaf9643d30) at os/filestore/WBThrottle.cc:139
#3 0x00007fbb05e06c32 in WBThrottle::entry (this=0x7fbb10f906f8) at os/filestore/WBThrottle.cc:157
#4 0x00007fbb03e1cdc5 in start_thread () from /lib64/libpthread.so.0
#5 0x00007fbb024a828d in ?? () from /lib64/libc.so.6
#6 0x0000000000000000 in ?? ()

Thread 22 (Thread 0x7fbaf563d700 (LWP 16986)):
#0 0x00007fbb03e206d5 in pthread_cond_wait@@GLIBC_2.3.2 () from /lib64/libpthread.so.0
#1 0x00007fbb06031378 in Wait (mutex=..., this=0x7fbb10f94480) at common/Cond.h:56
#2 Finisher::finisher_thread_entry (this=0x7fbb10f94420) at common/Finisher.cc:96
#3 0x00007fbb03e1cdc5 in start_thread () from /lib64/libpthread.so.0
#4 0x00007fbb024a828d in ?? () from /lib64/libc.so.6
#5 0x0000000000000000 in ?? ()

Thread 21 (Thread 0x7fbaeac92700 (LWP 17012)):
#0 0x00007fbb03e20a82 in pthread_cond_timedwait@@GLIBC_2.3.2 () from /lib64/libpthread.so.0
#1 0x00007fbb060f43da in WaitUntil (when=..., mutex=..., this=0x7fbb1114a650) at common/Cond.h:72
#2 WaitInterval (interval=..., mutex=..., cct=<optimized out>, this=0x7fbb1114a650) at common/Cond.h:81
#3 ThreadPool::worker (this=0x7fbb1114a5d0, wt=0x7fbb11192700) at common/WorkQueue.cc:151
#4 0x00007fbb060f5670 in ThreadPool::WorkThread::entry (this=<optimized out>) at common/WorkQueue.h:445
#5 0x00007fbb03e1cdc5 in start_thread () from /lib64/libpthread.so.0
#6 0x00007fbb024a828d in ?? () from /lib64/libc.so.6
#7 0x0000000000000000 in ?? ()

Thread 20 (Thread 0x7fbaf8643700 (LWP 16980)):
#0 0x00007fbb03e206d5 in pthread_cond_wait@@GLIBC_2.3.2 () from /lib64/libpthread.so.0
#1 0x00007fbb05aac57e in Cond::Wait (this=0x7fbb11158740, mutex=...) at common/Cond.h:56
#2 0x00007fbb05eac3d4 in FileJournal::write_thread_entry (this=0x7fbb11158600) at os/filestore/FileJournal.cc:1273
#3 0x00007fbb05de754d in FileJournal::Writer::entry (this=<optimized out>) at os/filestore/FileJournal.h:372
#4 0x00007fbb03e1cdc5 in start_thread () from /lib64/libpthread.so.0
#5 0x00007fbb024a828d in ?? () from /lib64/libc.so.6
#6 0x0000000000000000 in ?? ()

Thread 19 (Thread 0x7fbaeec9a700 (LWP 17004)):
#0 0x00007fbb03e206d5 in pthread_cond_wait@@GLIBC_2.3.2 () from /lib64/libpthread.so.0
#1 0x00007fbb061c535f in Wait (mutex=..., this=0x7fbb10fa4ae8) at common/Cond.h:56
#2 DispatchQueue::run_local_delivery (this=0x7fbb10fa48d0) at msg/simple/DispatchQueue.cc:114
#3 0x00007fbb060def5d in DispatchQueue::LocalDeliveryThread::entry (this=<optimized out>) at msg/simple/DispatchQueue.h:117
#4 0x00007fbb03e1cdc5 in start_thread () from /lib64/libpthread.so.0
#5 0x00007fbb024a828d in ?? () from /lib64/libc.so.6
#6 0x0000000000000000 in ?? ()

Thread 18 (Thread 0x7fbaf3639700 (LWP 16990)):
#0 0x00007fbb03e20a82 in pthread_cond_timedwait@@GLIBC_2.3.2 () from /lib64/libpthread.so.0
#1 0x00007fbb060ec99e in WaitUntil (when=..., mutex=..., this=0x7fbb10f905d8) at common/Cond.h:72
#2 SafeTimer::timer_thread (this=0x7fbb10f905c8) at common/Timer.cc:118
#3 0x00007fbb060edd3d in SafeTimerThread::entry (this=<optimized out>) at common/Timer.cc:38
#4 0x00007fbb03e1cdc5 in start_thread () from /lib64/libpthread.so.0
#5 0x00007fbb024a828d in ?? () from /lib64/libc.so.6
---Type <return> to continue, or q <return> to quit---
#6 0x0000000000000000 in ?? ()

Thread 17 (Thread 0x7fbaec495700 (LWP 17009)):
#0 0x00007fbb03e22f4d in __lll_lock_wait () from /lib64/libpthread.so.0
#1 0x00007fbb03e254bc in _L_cond_lock_792 () from /lib64/libpthread.so.0
#2 0x00007fbb03e25398 in __pthread_mutex_cond_lock () from /lib64/libpthread.so.0
#3 0x00007fbb03e20b54 in pthread_cond_timedwait@@GLIBC_2.3.2 () from /lib64/libpthread.so.0
#4 0x00007fbb060ec99e in WaitUntil (when=..., mutex=..., this=0x7ffd681759c8) at common/Cond.h:72
#5 SafeTimer::timer_thread (this=0x7ffd681759b8) at common/Timer.cc:118
#6 0x00007fbb060edd3d in SafeTimerThread::entry (this=<optimized out>) at common/Timer.cc:38
#7 0x00007fbb03e1cdc5 in start_thread () from /lib64/libpthread.so.0
#8 0x00007fbb024a828d in ?? () from /lib64/libc.so.6
#9 0x0000000000000000 in ?? ()

Thread 16 (Thread 0x7fbafa647700 (LWP 16951)):
#0 0x00007fbb03e22f4d in __lll_lock_wait () from /lib64/libpthread.so.0
#1 0x00007fbb03e1ed02 in _L_lock_791 () from /lib64/libpthread.so.0
#2 0x00007fbb03e1ec08 in pthread_mutex_lock () from /lib64/libpthread.so.0
#3 0x00007fbb060b47d8 in Mutex::Lock (this=this@entry=0x7fbb10fa44b0, no_lockdep=no_lockdep@entry=false) at common/Mutex.cc:110
#4 0x00007fbb060d8b04 in SimpleMessenger::mark_down (this=0x7fbb10fa4000, con=0x7fbb10f4a880) at msg/simple/SimpleMessenger.cc:655
#5 0x00007fbb060a4e4f in MonClient::_reopen_session (this=this@entry=0x7ffd681757d0, rank=rank@entry=-1, name="") at mon/MonClient.cc:617
#6 0x00007fbb05a6f9bb in _reopen_session (this=0x7ffd681757d0) at mon/MonClient.h:191
#7 reopen_session (cb=0x0, this=0x7ffd681757d0) at mon/MonClient.h:346
#8 OSD::tick_without_osd_lock (this=0x7fbb1114a000) at osd/OSD.cc:4298
#9 0x00007fbb05aa3e29 in Context::complete (this=0x7fbb10eac3b0, r=<optimized out>) at include/Context.h:64
#10 0x00007fbb060ec304 in SafeTimer::timer_thread (this=0x7fbb1114a190) at common/Timer.cc:105
#11 0x00007fbb060edd3d in SafeTimerThread::entry (this=<optimized out>) at common/Timer.cc:38
#12 0x00007fbb03e1cdc5 in start_thread () from /lib64/libpthread.so.0
#13 0x00007fbb024a828d in ?? () from /lib64/libc.so.6
#14 0x0000000000000000 in ?? ()

Thread 15 (Thread 0x7fbaf5e3e700 (LWP 16985)):
#0 0x00007fbb03e206d5 in pthread_cond_wait@@GLIBC_2.3.2 () from /lib64/libpthread.so.0
#1 0x00007fbb06031378 in Wait (mutex=..., this=0x7fbb10f94320) at common/Cond.h:56
#2 Finisher::finisher_thread_entry (this=0x7fbb10f942c0) at common/Finisher.cc:96
#3 0x00007fbb03e1cdc5 in start_thread () from /lib64/libpthread.so.0
#4 0x00007fbb024a828d in ?? () from /lib64/libc.so.6
#5 0x0000000000000000 in ?? ()

Thread 14 (Thread 0x7fbaf8e44700 (LWP 16958)):
#0 0x00007fbb03e20a82 in pthread_cond_timedwait@@GLIBC_2.3.2 () from /lib64/libpthread.so.0
#1 0x00007fbb05de858d in WaitUntil (when=..., mutex=..., this=0x7fbb10f90530) at common/Cond.h:72
#2 Cond::WaitInterval (this=this@entry=0x7fbb10f90530, cct=<optimized out>, mutex=..., interval=...) at common/Cond.h:81
#3 0x00007fbb05dc3f7d in FileStore::sync_entry (this=0x7fbb10f90000) at os/filestore/FileStore.cc:3690
#4 0x00007fbb05def3ed in FileStore::SyncThread::entry (this=<optimized out>) at os/filestore/FileStore.h:185
#5 0x00007fbb03e1cdc5 in start_thread () from /lib64/libpthread.so.0
#6 0x00007fbb024a828d in ?? () from /lib64/libc.so.6
#7 0x0000000000000000 in ?? ()

Thread 13 (Thread 0x7fbaf0c9e700 (LWP 17000)):
#0 0x00007fbb03e22f4d in __lll_lock_wait () from /lib64/libpthread.so.0
#1 0x00007fbb03e1ed02 in _L_lock_791 () from /lib64/libpthread.so.0
#2 0x00007fbb03e1ec08 in pthread_mutex_lock () from /lib64/libpthread.so.0
#3 0x00007fbb060b47d8 in Mutex::Lock (this=this@entry=0x7fbb1345d340, no_lockdep=no_lockdep@entry=false) at common/Mutex.cc:110
#4 0x00007fbb05a55665 in Locker (m=..., this=<synthetic pointer>) at common/Mutex.h:115
#5 session_handle_reset (session=0x7fbb1345d260, this=0x7fbb1114a000) at osd/OSD.h:1438
#6 OSD::ms_handle_reset (this=0x7fbb1114a000, con=0x7fbb13187680) at osd/OSD.cc:4687
#7 0x00007fbb061c560f in ms_deliver_handle_reset (con=0x7fbb13187680, this=0x7fbb10fa4000) at msg/Messenger.h:657
#8 DispatchQueue::entry (this=0x7fbb10fa41d0) at msg/simple/DispatchQueue.cc:173
#9 0x00007fbb060def3d in DispatchQueue::DispatchThread::entry (this=<optimized out>) at msg/simple/DispatchQueue.h:103
#10 0x00007fbb03e1cdc5 in start_thread () from /lib64/libpthread.so.0
#11 0x00007fbb024a828d in ?? () from /lib64/libc.so.6
#12 0x0000000000000000 in ?? ()

Thread 12 (Thread 0x7fbafb649700 (LWP 16949)):
#0 0x00007fbb03e206d5 in pthread_cond_wait@@GLIBC_2.3.2 () from /lib64/libpthread.so.0
#1 0x00007fbb060da13f in Wait (mutex=..., this=0x7fbb10fa4d60) at common/Cond.h:56
#2 SimpleMessenger::reaper_entry (this=0x7fbb10fa4700) at msg/simple/SimpleMessenger.cc:217
#3 0x00007fbb060dfa7d in SimpleMessenger::ReaperThread::entry (this=<optimized out>) at msg/simple/SimpleMessenger.h:196
#4 0x00007fbb03e1cdc5 in start_thread () from /lib64/libpthread.so.0
#5 0x00007fbb024a828d in ?? () from /lib64/libc.so.6
#6 0x0000000000000000 in ?? ()

Thread 11 (Thread 0x7fbaefc9c700 (LWP 17002)):
#0 0x00007fbb0249dc3d in __statvfs_getflags () from /lib64/libc.so.6
#1 0x0000000000000000 in ?? ()

Thread 10 (Thread 0x7fbafc64b700 (LWP 16947)):
#0 0x00007fbb03e22f4d in __lll_lock_wait () from /lib64/libpthread.so.0
#1 0x00007fbb03e1ed02 in _L_lock_791 () from /lib64/libpthread.so.0
#2 0x00007fbb03e1ec08 in pthread_mutex_lock () from /lib64/libpthread.so.0
#3 0x00007fbb060b47d8 in Mutex::Lock (this=this@entry=0x7fbb1114ae30, no_lockdep=no_lockdep@entry=false) at common/Mutex.cc:110
#4 0x00007fbb05a56b60 in OSD::heartbeat_reset (this=0x7fbb1114a000, con=0x7fbb13a60000) at osd/OSD.cc:4147
#5 0x00007fbb0622aac5 in ms_deliver_handle_reset (con=0x7fbb13a60000, this=0x7fbb10fac000) at msg/Messenger.h:657
#6 C_handle_reset::do_request (this=<optimized out>, id=<optimized out>) at msg/async/AsyncConnection.cc:87
#7 0x00007fbb061ca09a in EventCenter::process_events (this=this@entry=0x7fbb10fa8a48, timeout_microseconds=<optimized out>, timeout_microseconds@entry=30000000) at msg/async/Event.cc:406
---Type <return> to continue, or q <return> to quit---
#8 0x00007fbb061a90e0 in Worker::entry (this=0x7fbb10fa8a00) at msg/async/AsyncMessenger.cc:294
#9 0x00007fbb03e1cdc5 in start_thread () from /lib64/libpthread.so.0
#10 0x00007fbb024a828d in ?? () from /lib64/libc.so.6
#11 0x0000000000000000 in ?? ()

Thread 9 (Thread 0x7fbaf9e46700 (LWP 16952)):
#0 0x00007fbb03e206d5 in pthread_cond_wait@@GLIBC_2.3.2 () from /lib64/libpthread.so.0
#1 0x00007fbb060ec6ff in Wait (mutex=..., this=0x7fbb1114c588) at common/Cond.h:56
#2 SafeTimer::timer_thread (this=0x7fbb1114c578) at common/Timer.cc:116
#3 0x00007fbb060edd3d in SafeTimerThread::entry (this=<optimized out>) at common/Timer.cc:38
#4 0x00007fbb03e1cdc5 in start_thread () from /lib64/libpthread.so.0
#5 0x00007fbb024a828d in ?? () from /lib64/libc.so.6
#6 0x0000000000000000 in ?? ()

Thread 8 (Thread 0x7fbafe64f700 (LWP 16943)):
#0 0x00007fbb03e22f4d in __lll_lock_wait () from /lib64/libpthread.so.0
#1 0x00007fbb03e1ed02 in _L_lock_791 () from /lib64/libpthread.so.0
#2 0x00007fbb03e1ec08 in pthread_mutex_lock () from /lib64/libpthread.so.0
#3 0x00007fbb060b47d8 in Mutex::Lock (this=this@entry=0x7ffd68175960, no_lockdep=no_lockdep@entry=false) at common/Mutex.cc:110
#4 0x00007fbb05cb7afa in Locker (m=..., this=<synthetic pointer>) at common/Mutex.h:115
#5 sub_want (flags=1, start=35, what="", this=0x7ffd681757d0) at mon/MonClient.h:264
#6 Objecter::_maybe_request_map (this=this@entry=0x7fbb11158000) at osdc/Objecter.cc:1860
#7 0x00007fbb05cbf386 in Objecter::tick (this=0x7fbb11158000) at osdc/Objecter.cc:2056
#8 0x00007fbb05ae26d1 in operator() (this=0x7fbb10ee6850) at /usr/include/c++/4.8.2/functional:2471
#9 ceph::timer_detail::timer<ceph::time_detail::mono_clock>::timer_thread (this=0x7fbb111581f8) at common/ceph_timer.h:137
#10 0x00007fbb02d40220 in ?? () from /lib64/libstdc++.so.6
#11 0x00007fbb03e1cdc5 in start_thread () from /lib64/libpthread.so.0
#12 0x00007fbb024a828d in ?? () from /lib64/libc.so.6
#13 0x0000000000000000 in ?? ()

Thread 7 (Thread 0x7fbaff9bc700 (LWP 16940)):
#0 0x00007fbb0249dc3d in __statvfs_getflags () from /lib64/libc.so.6
#1 0x0000000000000000 in ?? ()

Thread 6 (Thread 0x7fbafce4c700 (LWP 16946)):
#0 0x00007fbb03e22f4d in __lll_lock_wait () from /lib64/libpthread.so.0
#1 0x00007fbb03e1ed02 in _L_lock_791 () from /lib64/libpthread.so.0
#2 0x00007fbb03e1ec08 in pthread_mutex_lock () from /lib64/libpthread.so.0
#3 0x00007fbb060b47d8 in Mutex::Lock (this=this@entry=0x7fbb1114ae30, no_lockdep=no_lockdep@entry=false) at common/Mutex.cc:110
#4 0x00007fbb05a76f6e in OSD::handle_osd_ping (this=this@entry=0x7fbb1114a000, m=m@entry=0x7fbb111ffc00) at osd/OSD.cc:3878
#5 0x00007fbb05a7866b in OSD::heartbeat_dispatch (this=0x7fbb1114a000, m=0x7fbb111ffc00) at osd/OSD.cc:5713
#6 0x00007fbb06213d9b in ms_fast_dispatch (m=<optimized out>, this=0x7fbb10fac000) at msg/Messenger.h:555
#7 AsyncConnection::DelayedDelivery::do_request (this=<optimized out>, id=<optimized out>) at msg/async/AsyncConnection.cc:2453
#8 0x00007fbb061c8626 in EventCenter::process_time_events (this=this@entry=0x7fbb10fa87c8) at msg/async/Event.cc:307
#9 0x00007fbb061c9d78 in EventCenter::process_events (this=this@entry=0x7fbb10fa87c8, timeout_microseconds=<optimized out>, timeout_microseconds@entry=30000000) at msg/async/Event.cc:392
#10 0x00007fbb061a90e0 in Worker::entry (this=0x7fbb10fa8780) at msg/async/AsyncMessenger.cc:294
#11 0x00007fbb03e1cdc5 in start_thread () from /lib64/libpthread.so.0
#12 0x00007fbb024a828d in ?? () from /lib64/libc.so.6
#13 0x0000000000000000 in ?? ()

Thread 5 (Thread 0x7fbafae48700 (LWP 16950)):
#0 0x00007fbb03e22f4d in __lll_lock_wait () from /lib64/libpthread.so.0
#1 0x00007fbb03e254bc in _L_cond_lock_792 () from /lib64/libpthread.so.0
#2 0x00007fbb03e25398 in __pthread_mutex_cond_lock () from /lib64/libpthread.so.0
#3 0x00007fbb03e20b54 in pthread_cond_timedwait@@GLIBC_2.3.2 () from /lib64/libpthread.so.0
#4 0x00007fbb060ec99e in WaitUntil (when=..., mutex=..., this=0x7fbb1114a080) at common/Cond.h:72
#5 SafeTimer::timer_thread (this=0x7fbb1114a070) at common/Timer.cc:118
#6 0x00007fbb060edd3d in SafeTimerThread::entry (this=<optimized out>) at common/Timer.cc:38
#7 0x00007fbb03e1cdc5 in start_thread () from /lib64/libpthread.so.0
#8 0x00007fbb024a828d in ?? () from /lib64/libc.so.6
#9 0x0000000000000000 in ?? ()

Thread 4 (Thread 0x7fbafbe4a700 (LWP 16948)):
#0 0x00007fbb03e206d5 in pthread_cond_wait@@GLIBC_2.3.2 () from /lib64/libpthread.so.0
#1 0x00007fbb060da13f in Wait (mutex=..., this=0x7fbb10fa5460) at common/Cond.h:56
#2 SimpleMessenger::reaper_entry (this=0x7fbb10fa4e00) at msg/simple/SimpleMessenger.cc:217
#3 0x00007fbb060dfa7d in SimpleMessenger::ReaperThread::entry (this=<optimized out>) at msg/simple/SimpleMessenger.h:196
#4 0x00007fbb03e1cdc5 in start_thread () from /lib64/libpthread.so.0
#5 0x00007fbb024a828d in ?? () from /lib64/libc.so.6
#6 0x0000000000000000 in ?? ()

Thread 3 (Thread 0x7fbafde4e700 (LWP 16944)):
#0 0x00007fbb03e206d5 in pthread_cond_wait@@GLIBC_2.3.2 () from /lib64/libpthread.so.0
#1 0x00007fbb060da13f in Wait (mutex=..., this=0x7fbb10fa4660) at common/Cond.h:56
#2 SimpleMessenger::reaper_entry (this=0x7fbb10fa4000) at msg/simple/SimpleMessenger.cc:217
#3 0x00007fbb060dfa7d in SimpleMessenger::ReaperThread::entry (this=<optimized out>) at msg/simple/SimpleMessenger.h:196
#4 0x00007fbb03e1cdc5 in start_thread () from /lib64/libpthread.so.0
#5 0x00007fbb024a828d in ?? () from /lib64/libc.so.6
#6 0x0000000000000000 in ?? ()

Thread 2 (Thread 0x7fbafd64d700 (LWP 16945)):
#0 0x00007fbb03e22f4d in __lll_lock_wait () from /lib64/libpthread.so.0
#1 0x00007fbb03e1ed02 in _L_lock_791 () from /lib64/libpthread.so.0
#2 0x00007fbb03e1ec08 in pthread_mutex_lock () from /lib64/libpthread.so.0
#3 0x00007fbb060b47d8 in Mutex::Lock (this=this@entry=0x7fbb1114ae30, no_lockdep=no_lockdep@entry=false) at common/Mutex.cc:110
#4 0x00007fbb05a76f6e in OSD::handle_osd_ping (this=this@entry=0x7fbb1114a000, m=m@entry=0x7fbb113b7600) at osd/OSD.cc:3878
---Type <return> to continue, or q <return> to quit---
#5 0x00007fbb05a7866b in OSD::heartbeat_dispatch (this=0x7fbb1114a000, m=0x7fbb113b7600) at osd/OSD.cc:5713
#6 0x00007fbb06213d9b in ms_fast_dispatch (m=<optimized out>, this=0x7fbb10fac000) at msg/Messenger.h:555
#7 AsyncConnection::DelayedDelivery::do_request (this=<optimized out>, id=<optimized out>) at msg/async/AsyncConnection.cc:2453
#8 0x00007fbb061c8626 in EventCenter::process_time_events (this=this@entry=0x7fbb10fa8548) at msg/async/Event.cc:307
#9 0x00007fbb061c9d78 in EventCenter::process_events (this=this@entry=0x7fbb10fa8548, timeout_microseconds=<optimized out>, timeout_microseconds@entry=30000000) at msg/async/Event.cc:392
#10 0x00007fbb061a90e0 in Worker::entry (this=0x7fbb10fa8500) at msg/async/AsyncMessenger.cc:294
#11 0x00007fbb03e1cdc5 in start_thread () from /lib64/libpthread.so.0
#12 0x00007fbb024a828d in ?? () from /lib64/libc.so.6
#13 0x0000000000000000 in ?? ()

Thread 1 (Thread 0x7fbb001bd700 (LWP 16939)):
#0 0x00007fbb03e23fcb in raise () from /lib64/libpthread.so.0
#1 0x00007fbb06006975 in reraise_fatal (signum=6) at global/signal_handler.cc:71
#2 handle_fatal_signal (signum=6) at global/signal_handler.cc:133
#3 <signal handler called>
#4 0x00007fbb023e75f7 in sigtimedwait () from /lib64/libc.so.6
#5 0x00007fbb001b95a8 in ?? ()
#6 0x0000000000000000 in ?? ()
(gdb)
(gdb)
    (1-1/1)