[New LWP 17032] [New LWP 18878] [New LWP 21174] [New LWP 17364] [Thread debugging using libthread_db enabled] Using host libthread_db library "/lib64/libthread_db.so.1". Core was generated by `ceph-osd -f --cluster ceph -i 0'. Program terminated with signal 6, Aborted. #0 0x00007fbb03e23fcb in raise () from /lib64/libpthread.so.0 Missing separate debuginfos, use: debuginfo-install boost-iostreams-1.53.0-25.el7.x86_64 boost-random-1.53.0-25.el7.x86_64 boost-system-1.53.0-25.el7.x86_64 boost-thread-1.53.0-25.el7.x86_64 bzip2-libs-1.0.6-13.el7.x86_64 fuse-libs-2.9.2-6.el7.x86_64 glibc-2.17-106.el7_2.8.x86_64 gperftools-libs-2.4-7.el7.x86_64 leveldb-1.12.0-11.el7.x86_64 libaio-0.3.109-13.el7.x86_64 libblkid-2.23.2-26.el7_2.3.x86_64 libgcc-4.8.5-4.el7.x86_64 libstdc++-4.8.5-4.el7.x86_64 libunwind-1.1-5.el7_2.2.x86_64 libuuid-2.23.2-26.el7_2.3.x86_64 lttng-ust-2.4.1-1.el7.x86_64 nspr-4.10.8-2.el7_1.x86_64 nss-3.19.1-18.el7.x86_64 nss-softokn-3.16.2.3-13.el7_1.x86_64 nss-softokn-freebl-3.16.2.3-13.el7_1.x86_64 nss-util-3.19.1-4.el7_1.x86_64 snappy-1.1.0-3.el7.x86_64 sqlite-3.7.17-8.el7.x86_64 userspace-rcu-0.7.16-1.el7.x86_64 zlib-1.2.7-15.el7.x86_64 (gdb) bt #0 0x00007fbb03e23fcb in raise () from /lib64/libpthread.so.0 #1 0x00007fbb06006975 in reraise_fatal (signum=6) at global/signal_handler.cc:71 #2 handle_fatal_signal (signum=6) at global/signal_handler.cc:133 #3 #4 0x00007fbb023e75f7 in sigtimedwait () from /lib64/libc.so.6 #5 0x00007fbb001b95a8 in ?? () #6 0x0000000000000000 in ?? () (gdb) thr app all bt Thread 87 (Thread 0x7fbad975d700 (LWP 17364)): #0 0x00007fbb03e206d5 in pthread_cond_wait@@GLIBC_2.3.2 () from /lib64/libpthread.so.0 #1 0x00007fbb061f60b9 in Wait (mutex=..., this=0x7fbb1137dab8) at common/Cond.h:56 #2 Pipe::DelayedDelivery::entry (this=0x7fbb1137d9e0) at msg/simple/Pipe.cc:203 #3 0x00007fbb03e1cdc5 in start_thread () from /lib64/libpthread.so.0 #4 0x00007fbb024a828d in ?? () from /lib64/libc.so.6 #5 0x0000000000000000 in ?? () Thread 86 (Thread 0x7fbad7b56700 (LWP 21174)): #0 0x00007fbb03e22f4d in __lll_lock_wait () from /lib64/libpthread.so.0 #1 0x00007fbb03e1ed02 in _L_lock_791 () from /lib64/libpthread.so.0 #2 0x00007fbb03e1ec08 in pthread_mutex_lock () from /lib64/libpthread.so.0 #3 0x00007fbb060b47d8 in Mutex::Lock (this=0x7fbb10fa44b0, no_lockdep=no_lockdep@entry=false) at common/Mutex.cc:110 #4 0x00007fbb061f8139 in Pipe::fault (this=this@entry=0x7fbb12d74800, onread=onread@entry=true) at msg/simple/Pipe.cc:1391 #5 0x00007fbb061fecc0 in Pipe::reader (this=0x7fbb12d74800) at msg/simple/Pipe.cc:1697 #6 0x00007fbb062084bd in Pipe::Reader::entry (this=) at msg/simple/Pipe.h:52 #7 0x00007fbb03e1cdc5 in start_thread () from /lib64/libpthread.so.0 #8 0x00007fbb024a828d in ?? () from /lib64/libc.so.6 #9 0x0000000000000000 in ?? () Thread 85 (Thread 0x7fbad8559700 (LWP 18878)): #0 0x00007fbb03e22f4d in __lll_lock_wait () from /lib64/libpthread.so.0 #1 0x00007fbb03e1ed02 in _L_lock_791 () from /lib64/libpthread.so.0 #2 0x00007fbb03e1ec08 in pthread_mutex_lock () from /lib64/libpthread.so.0 #3 0x00007fbb060b47d8 in Mutex::Lock (this=0x7fbb10fa44b0, no_lockdep=no_lockdep@entry=false) at common/Mutex.cc:110 #4 0x00007fbb061f8139 in Pipe::fault (this=this@entry=0x7fbb112e3400, onread=onread@entry=true) at msg/simple/Pipe.cc:1391 #5 0x00007fbb061fecc0 in Pipe::reader (this=0x7fbb112e3400) at msg/simple/Pipe.cc:1697 #6 0x00007fbb062084bd in Pipe::Reader::entry (this=) at msg/simple/Pipe.h:52 #7 0x00007fbb03e1cdc5 in start_thread () from /lib64/libpthread.so.0 #8 0x00007fbb024a828d in ?? () from /lib64/libc.so.6 #9 0x0000000000000000 in ?? () Thread 84 (Thread 0x7fbb056b9700 (LWP 17032)): #0 0x00007fbb03e206d5 in pthread_cond_wait@@GLIBC_2.3.2 () from /lib64/libpthread.so.0 #1 0x00007fbb062055cf in Wait (mutex=..., this=0x7fbb111bb248) at common/Cond.h:56 #2 Pipe::writer (this=0x7fbb111ba000) at msg/simple/Pipe.cc:1864 #3 0x00007fbb062084dd in Pipe::Writer::entry (this=) at msg/simple/Pipe.h:63 #4 0x00007fbb03e1cdc5 in start_thread () from /lib64/libpthread.so.0 #5 0x00007fbb024a828d in ?? () from /lib64/libc.so.6 #6 0x0000000000000000 in ?? () Thread 83 (Thread 0x7fbad8e5b700 (LWP 19777)): #0 0x00007fbb03e206d5 in pthread_cond_wait@@GLIBC_2.3.2 () from /lib64/libpthread.so.0 #1 0x00007fbb061ff0ad in Wait (mutex=..., this=0x7fbb117bd248) at common/Cond.h:56 #2 Pipe::reader (this=0x7fbb117bc000) at msg/simple/Pipe.cc:1529 #3 0x00007fbb062084bd in Pipe::Reader::entry (this=) at msg/simple/Pipe.h:52 #4 0x00007fbb03e1cdc5 in start_thread () from /lib64/libpthread.so.0 #5 0x00007fbb024a828d in ?? () from /lib64/libc.so.6 #6 0x0000000000000000 in ?? () Thread 82 (Thread 0x7fbadbe68700 (LWP 21078)): #0 0x00007fbb03e206d5 in pthread_cond_wait@@GLIBC_2.3.2 () from /lib64/libpthread.so.0 #1 0x00007fbb062055cf in Wait (mutex=..., this=0x7fbb117be648) at common/Cond.h:56 #2 Pipe::writer (this=0x7fbb117bd400) at msg/simple/Pipe.cc:1864 #3 0x00007fbb062084dd in Pipe::Writer::entry (this=) at msg/simple/Pipe.h:63 #4 0x00007fbb03e1cdc5 in start_thread () from /lib64/libpthread.so.0 #5 0x00007fbb024a828d in ?? () from /lib64/libc.so.6 #6 0x0000000000000000 in ?? () Thread 81 (Thread 0x7fbae8c8e700 (LWP 17016)): #0 0x00007fbb03e20a82 in pthread_cond_timedwait@@GLIBC_2.3.2 () from /lib64/libpthread.so.0 #1 0x00007fbb05a7a6e5 in WaitUntil (when=..., mutex=..., this=0x7fbb10edca58) at common/Cond.h:72 #2 WaitInterval (interval=..., mutex=..., cct=, this=0x7fbb10edca58) at common/Cond.h:81 #3 OSD::ShardedOpWQ::_process (this=0x7fbb1114b1e8, thread_index=, hb=0x7fbb111935d0) at osd/OSD.cc:8644 #4 0x00007fbb060f36e7 in ShardedThreadPool::shardedthreadpool_worker (this=0x7fbb1114a768, thread_index=) at common/WorkQueue.cc:340 #5 0x00007fbb060f5650 in ShardedThreadPool::WorkThreadSharded::entry (this=) at common/WorkQueue.h:684 #6 0x00007fbb03e1cdc5 in start_thread () from /lib64/libpthread.so.0 #7 0x00007fbb024a828d in ?? () from /lib64/libc.so.6 #8 0x0000000000000000 in ?? () Thread 80 (Thread 0x7fbada05f700 (LWP 17354)): #0 0x00007fbb03e206d5 in pthread_cond_wait@@GLIBC_2.3.2 () from /lib64/libpthread.so.0 #1 0x00007fbb061f60b9 in Wait (mutex=..., this=0x7fbb1137c438) at common/Cond.h:56 #2 Pipe::DelayedDelivery::entry (this=0x7fbb1137c360) at msg/simple/Pipe.cc:203 #3 0x00007fbb03e1cdc5 in start_thread () from /lib64/libpthread.so.0 #4 0x00007fbb024a828d in ?? () from /lib64/libc.so.6 #5 0x0000000000000000 in ?? () Thread 79 (Thread 0x7fbb01131700 (LWP 16928)): #0 0x00007fbb03e206d5 in pthread_cond_wait@@GLIBC_2.3.2 () from /lib64/libpthread.so.0 #1 0x00007fbb0604b74b in ceph::log::Log::entry (this=0x7fbb10f49800) at log/Log.cc:457 #2 0x00007fbb03e1cdc5 in start_thread () from /lib64/libpthread.so.0 #3 0x00007fbb024a828d in ?? () from /lib64/libc.so.6 #4 0x0000000000000000 in ?? () ---Type to continue, or q to quit--- Thread 78 (Thread 0x7fbae4485700 (LWP 17025)): #0 0x00007fbb03e20a82 in pthread_cond_timedwait@@GLIBC_2.3.2 () from /lib64/libpthread.so.0 #1 0x00007fbb060f43da in WaitUntil (when=..., mutex=..., this=0x7fbb1114ac58) at common/Cond.h:72 #2 WaitInterval (interval=..., mutex=..., cct=, this=0x7fbb1114ac58) at common/Cond.h:81 #3 ThreadPool::worker (this=0x7fbb1114abd8, wt=0x7fbb11192910) at common/WorkQueue.cc:151 #4 0x00007fbb060f5670 in ThreadPool::WorkThread::entry (this=) at common/WorkQueue.h:445 #5 0x00007fbb03e1cdc5 in start_thread () from /lib64/libpthread.so.0 #6 0x00007fbb024a828d in ?? () from /lib64/libc.so.6 #7 0x0000000000000000 in ?? () Thread 77 (Thread 0x7fbad7a55700 (LWP 22006)): #0 0x00007fbb03e206d5 in pthread_cond_wait@@GLIBC_2.3.2 () from /lib64/libpthread.so.0 #1 0x00007fbb061ff0ad in Wait (mutex=..., this=0x7fbb11297248) at common/Cond.h:56 #2 Pipe::reader (this=0x7fbb11296000) at msg/simple/Pipe.cc:1529 #3 0x00007fbb062084bd in Pipe::Reader::entry (this=) at msg/simple/Pipe.h:52 #4 0x00007fbb03e1cdc5 in start_thread () from /lib64/libpthread.so.0 #5 0x00007fbb024a828d in ?? () from /lib64/libc.so.6 #6 0x0000000000000000 in ?? () Thread 76 (Thread 0x7fbadd16d700 (LWP 17308)): #0 0x00007fbb024a8863 in open_by_handle_at () from /lib64/libc.so.6 #1 0x0000000000000000 in ?? () Thread 75 (Thread 0x7fbad7954700 (LWP 22008)): #0 0x00007fbb03e206d5 in pthread_cond_wait@@GLIBC_2.3.2 () from /lib64/libpthread.so.0 #1 0x00007fbb061f60b9 in Wait (mutex=..., this=0x7fbb112fcaf8) at common/Cond.h:56 #2 Pipe::DelayedDelivery::entry (this=0x7fbb112fca20) at msg/simple/Pipe.cc:203 #3 0x00007fbb03e1cdc5 in start_thread () from /lib64/libpthread.so.0 #4 0x00007fbb024a828d in ?? () from /lib64/libc.so.6 #5 0x0000000000000000 in ?? () Thread 74 (Thread 0x7fbae4c86700 (LWP 17024)): #0 0x00007fbb03e20a82 in pthread_cond_timedwait@@GLIBC_2.3.2 () from /lib64/libpthread.so.0 #1 0x00007fbb060f43da in WaitUntil (when=..., mutex=..., this=0x7fbb1114aac0) at common/Cond.h:72 #2 WaitInterval (interval=..., mutex=..., cct=, this=0x7fbb1114aac0) at common/Cond.h:81 #3 ThreadPool::worker (this=0x7fbb1114aa40, wt=0x7fbb111928b0) at common/WorkQueue.cc:151 #4 0x00007fbb060f5670 in ThreadPool::WorkThread::entry (this=) at common/WorkQueue.h:445 #5 0x00007fbb03e1cdc5 in start_thread () from /lib64/libpthread.so.0 #6 0x00007fbb024a828d in ?? () from /lib64/libc.so.6 #7 0x0000000000000000 in ?? () Thread 73 (Thread 0x7fbb056cc800 (LWP 16927)): #0 0x00007fbb03e1def7 in pthread_join () from /lib64/libpthread.so.0 #1 0x00007fbb060e6310 in Thread::join (this=this@entry=0x7fbb10fa4440, prval=prval@entry=0x0) at common/Thread.cc:171 #2 0x00007fbb061c3f52 in DispatchQueue::wait (this=this@entry=0x7fbb10fa41d0) at msg/simple/DispatchQueue.cc:226 #3 0x00007fbb060dab5b in SimpleMessenger::wait (this=0x7fbb10fa4000) at msg/simple/SimpleMessenger.cc:543 #4 0x00007fbb059f60f6 in main (argc=, argv=) at ceph_osd.cc:623 Thread 72 (Thread 0x7fbad7053700 (LWP 21307)): #0 0x00007fbb03e22f4d in __lll_lock_wait () from /lib64/libpthread.so.0 #1 0x00007fbb03e1ed02 in _L_lock_791 () from /lib64/libpthread.so.0 #2 0x00007fbb03e1ec08 in pthread_mutex_lock () from /lib64/libpthread.so.0 #3 0x00007fbb060b47d8 in Mutex::Lock (this=this@entry=0x7fbb13d0d170, no_lockdep=no_lockdep@entry=false) at common/Mutex.cc:110 #4 0x00007fbb061ff791 in Pipe::reader (this=0x7fbb13d0c000) at msg/simple/Pipe.cc:1606 #5 0x00007fbb062084bd in Pipe::Reader::entry (this=) at msg/simple/Pipe.h:52 #6 0x00007fbb03e1cdc5 in start_thread () from /lib64/libpthread.so.0 #7 0x00007fbb024a828d in ?? () from /lib64/libc.so.6 #8 0x0000000000000000 in ?? () Thread 71 (Thread 0x7fbae7c8c700 (LWP 17018)): #0 0x00007fbb03e22f4d in __lll_lock_wait () from /lib64/libpthread.so.0 #1 0x00007fbb03e1ed02 in _L_lock_791 () from /lib64/libpthread.so.0 #2 0x00007fbb03e1ec08 in pthread_mutex_lock () from /lib64/libpthread.so.0 #3 0x00007fbb060b47d8 in Mutex::Lock (this=this@entry=0x7fbb13d0d170, no_lockdep=no_lockdep@entry=false) at common/Mutex.cc:110 #4 0x00007fbb060d85d6 in Locker (m=..., this=) at common/Mutex.h:115 #5 is_connected (this=0x7fbb13d0c000) at msg/simple/Pipe.h:143 #6 SimpleMessenger::is_connected (this=, con=) at msg/simple/SimpleMessenger.cc:282 #7 0x00007fbb05a4dc06 in OSD::op_is_discardable (op=op@entry=0x7fbb12215b80) at osd/OSD.cc:8616 #8 0x00007fbb05b22cec in PG::can_discard_op (this=0x7fbb1142d000, op=std::shared_ptr (count 3, weak 0) 0x7fbb1144fa00) at osd/PG.cc:5353 #9 0x00007fbb05b23405 in PG::can_discard_request (this=this@entry=0x7fbb1142d000, op=std::shared_ptr (count 3, weak 0) 0x7fbb1144fa00) at osd/PG.cc:5439 #10 0x00007fbb05bc21e9 in ReplicatedPG::do_request (this=0x7fbb1142d000, op=std::shared_ptr (count 3, weak 0) 0x7fbb1144fa00, handle=...) at osd/ReplicatedPG.cc:1448 #11 0x00007fbb05a7607d in OSD::dequeue_op (this=0x7fbb1114a000, pg=..., op=std::shared_ptr (count 3, weak 0) 0x7fbb1144fa00, handle=...) at osd/OSD.cc:8815 #12 0x00007fbb05a762cd in PGQueueable::RunVis::operator() (this=this@entry=0x7fbae7c8a7b0, op=std::shared_ptr (count 3, weak 0) 0x7fbb1144fa00) at osd/OSD.cc:163 #13 0x00007fbb05a7adf9 in internal_visit > (operand=std::shared_ptr (count 3, weak 0) 0x7fbb1144fa00, this=) at /usr/include/boost/variant/variant.hpp:1017 #14 visitation_impl_invoke_impl, void*, std::shared_ptr > (storage=0x7fbae7c8a8d0, visitor=) at /usr/include/boost/variant/detail/visitation_impl.hpp:130 #15 visitation_impl_invoke, void*, std::shared_ptr, boost::variant, PGSnapTrim, PGScrub>::has_fallback_type_> (internal_which=, t=0x0, storage=0x7fbae7c8a8d0, visitor=) at /usr/include/boost/variant/detail/visitation_impl.hpp:173 #16 visitation_impl, boost::detail::variant::visitation_impl_step, std::shared_ptr, boost::mpl::l_item, PGSnapTrim, boost::mpl::l_item, PGScrub, boost::mpl::l_end> > > >, boost::mpl::l_iter >, boost::detail::variant::invoke_visitor, void*, boost::variant, PGSnapTrim, PGScrub>::has_fallback_type_> (no_backup_flag=..., storage=0x7fbae7c8a8d0, visitor=, logical_which=, internal_which=) at /usr/include/boost/variant/detail/visitation_impl.hpp:256 #17 internal_apply_visitor_impl, void*> (storage=0x7fbae7c8a8d0, visitor=, logical_which=, internal_which=) at /usr/include/boost/variant/variant.hpp:2326 #18 internal_apply_visitor > (visitor=, this=0x7fbae7c8a8c8) at /usr/include/boost/variant/variant.hpp:2337 #19 apply_visitor (visitor=..., this=0x7fbae7c8a8c8) at /usr/include/boost/variant/variant.hpp:2360 #20 apply_visitor, PGSnapTrim, PGScrub> > (visitable=..., visitor=...) at /usr/include/boost/variant/detail/apply_visitor_unary.hpp:60 #21 run (handle=..., pg=..., osd=, this=0x7fbae7c8a8c8) at osd/OSD.h:392 #22 OSD::ShardedOpWQ::_process (this=0x7fbb1114b1e8, thread_index=, hb=) at osd/OSD.cc:8696 ---Type to continue, or q to quit--- #23 0x00007fbb060f36e7 in ShardedThreadPool::shardedthreadpool_worker (this=0x7fbb1114a768, thread_index=) at common/WorkQueue.cc:340 #24 0x00007fbb060f5650 in ShardedThreadPool::WorkThreadSharded::entry (this=) at common/WorkQueue.h:684 #25 0x00007fbb03e1cdc5 in start_thread () from /lib64/libpthread.so.0 #26 0x00007fbb024a828d in ?? () from /lib64/libc.so.6 #27 0x0000000000000000 in ?? () Thread 70 (Thread 0x7fbade371700 (LWP 21176)): #0 0x00007fbb03e206d5 in pthread_cond_wait@@GLIBC_2.3.2 () from /lib64/libpthread.so.0 #1 0x00007fbb061f60b9 in Wait (mutex=..., this=0x7fbb112fd518) at common/Cond.h:56 #2 Pipe::DelayedDelivery::entry (this=0x7fbb112fd440) at msg/simple/Pipe.cc:203 #3 0x00007fbb03e1cdc5 in start_thread () from /lib64/libpthread.so.0 #4 0x00007fbb024a828d in ?? () from /lib64/libc.so.6 #5 0x0000000000000000 in ?? () Thread 69 (Thread 0x7fbae5487700 (LWP 17023)): #0 0x00007fbb03e20a82 in pthread_cond_timedwait@@GLIBC_2.3.2 () from /lib64/libpthread.so.0 #1 0x00007fbb060f43da in WaitUntil (when=..., mutex=..., this=0x7fbb1114a928) at common/Cond.h:72 #2 WaitInterval (interval=..., mutex=..., cct=, this=0x7fbb1114a928) at common/Cond.h:81 #3 ThreadPool::worker (this=0x7fbb1114a8a8, wt=0x7fbb111927f0) at common/WorkQueue.cc:151 #4 0x00007fbb060f5670 in ThreadPool::WorkThread::entry (this=) at common/WorkQueue.h:445 #5 0x00007fbb03e1cdc5 in start_thread () from /lib64/libpthread.so.0 #6 0x00007fbb024a828d in ?? () from /lib64/libc.so.6 #7 0x0000000000000000 in ?? () Thread 68 (Thread 0x7fbaed497700 (LWP 17007)): #0 0x00007fbb03e206d5 in pthread_cond_wait@@GLIBC_2.3.2 () from /lib64/libpthread.so.0 #1 0x00007fbb061c535f in Wait (mutex=..., this=0x7fbb10fa51e8) at common/Cond.h:56 #2 DispatchQueue::run_local_delivery (this=0x7fbb10fa4fd0) at msg/simple/DispatchQueue.cc:114 #3 0x00007fbb060def5d in DispatchQueue::LocalDeliveryThread::entry (this=) at msg/simple/DispatchQueue.h:117 #4 0x00007fbb03e1cdc5 in start_thread () from /lib64/libpthread.so.0 #5 0x00007fbb024a828d in ?? () from /lib64/libc.so.6 #6 0x0000000000000000 in ?? () Thread 67 (Thread 0x7fbade674700 (LWP 19778)): #0 0x00007fbb03e206d5 in pthread_cond_wait@@GLIBC_2.3.2 () from /lib64/libpthread.so.0 #1 0x00007fbb062055cf in Wait (mutex=..., this=0x7fbb117bd248) at common/Cond.h:56 #2 Pipe::writer (this=0x7fbb117bc000) at msg/simple/Pipe.cc:1864 #3 0x00007fbb062084dd in Pipe::Writer::entry (this=) at msg/simple/Pipe.h:63 #4 0x00007fbb03e1cdc5 in start_thread () from /lib64/libpthread.so.0 #5 0x00007fbb024a828d in ?? () from /lib64/libc.so.6 #6 0x0000000000000000 in ?? () Thread 66 (Thread 0x7fbaddb70700 (LWP 22726)): #0 0x00007fbb0249dc3d in __statvfs_getflags () from /lib64/libc.so.6 #1 0x0000000000000000 in ?? () Thread 65 (Thread 0x7fbae748b700 (LWP 17019)): #0 0x00007fbb03e20a82 in pthread_cond_timedwait@@GLIBC_2.3.2 () from /lib64/libpthread.so.0 #1 0x00007fbb05a7a6e5 in WaitUntil (when=..., mutex=..., this=0x7fbb10edc7d8) at common/Cond.h:72 #2 WaitInterval (interval=..., mutex=..., cct=, this=0x7fbb10edc7d8) at common/Cond.h:81 #3 OSD::ShardedOpWQ::_process (this=0x7fbb1114b1e8, thread_index=, hb=0x7fbb11193660) at osd/OSD.cc:8644 #4 0x00007fbb060f36e7 in ShardedThreadPool::shardedthreadpool_worker (this=0x7fbb1114a768, thread_index=) at common/WorkQueue.cc:340 #5 0x00007fbb060f5650 in ShardedThreadPool::WorkThreadSharded::entry (this=) at common/WorkQueue.h:684 #6 0x00007fbb03e1cdc5 in start_thread () from /lib64/libpthread.so.0 #7 0x00007fbb024a828d in ?? () from /lib64/libc.so.6 #8 0x0000000000000000 in ?? () Thread 64 (Thread 0x7fbaf049d700 (LWP 17001)): #0 0x00007fbb03e206d5 in pthread_cond_wait@@GLIBC_2.3.2 () from /lib64/libpthread.so.0 #1 0x00007fbb061c535f in Wait (mutex=..., this=0x7fbb10fa43e8) at common/Cond.h:56 #2 DispatchQueue::run_local_delivery (this=0x7fbb10fa41d0) at msg/simple/DispatchQueue.cc:114 #3 0x00007fbb060def5d in DispatchQueue::LocalDeliveryThread::entry (this=) at msg/simple/DispatchQueue.h:117 #4 0x00007fbb03e1cdc5 in start_thread () from /lib64/libpthread.so.0 #5 0x00007fbb024a828d in ?? () from /lib64/libc.so.6 #6 0x0000000000000000 in ?? () Thread 63 (Thread 0x7fbadbd67700 (LWP 17324)): #0 0x00007fbb03e206d5 in pthread_cond_wait@@GLIBC_2.3.2 () from /lib64/libpthread.so.0 #1 0x00007fbb061f60b9 in Wait (mutex=..., this=0x7fbb1104b5b8) at common/Cond.h:56 #2 Pipe::DelayedDelivery::entry (this=0x7fbb1104b4e0) at msg/simple/Pipe.cc:203 #3 0x00007fbb03e1cdc5 in start_thread () from /lib64/libpthread.so.0 #4 0x00007fbb024a828d in ?? () from /lib64/libc.so.6 #5 0x0000000000000000 in ?? () Thread 62 (Thread 0x7fbad985e700 (LWP 21074)): #0 0x00007fbb03e206d5 in pthread_cond_wait@@GLIBC_2.3.2 () from /lib64/libpthread.so.0 #1 0x00007fbb062055cf in Wait (mutex=..., this=0x7fbb11288648) at common/Cond.h:56 #2 Pipe::writer (this=0x7fbb11287400) at msg/simple/Pipe.cc:1864 #3 0x00007fbb062084dd in Pipe::Writer::entry (this=) at msg/simple/Pipe.h:63 #4 0x00007fbb03e1cdc5 in start_thread () from /lib64/libpthread.so.0 #5 0x00007fbb024a828d in ?? () from /lib64/libc.so.6 #6 0x0000000000000000 in ?? () Thread 61 (Thread 0x7fbadd96e700 (LWP 21309)): #0 0x00007fbb03e22f4d in __lll_lock_wait () from /lib64/libpthread.so.0 #1 0x00007fbb03e1ed02 in _L_lock_791 () from /lib64/libpthread.so.0 ---Type to continue, or q to quit--- #2 0x00007fbb03e1ec08 in pthread_mutex_lock () from /lib64/libpthread.so.0 #3 0x00007fbb060b47d8 in Mutex::Lock (this=this@entry=0x7fbb13d0d170, no_lockdep=no_lockdep@entry=false) at common/Mutex.cc:110 #4 0x00007fbb060d85d6 in Locker (m=..., this=) at common/Mutex.h:115 #5 is_connected (this=0x7fbb13d0c000) at msg/simple/Pipe.h:143 #6 SimpleMessenger::is_connected (this=, con=) at msg/simple/SimpleMessenger.cc:282 #7 0x00007fbb05a4dc06 in OSD::op_is_discardable (op=op@entry=0x7fbb12214dc0) at osd/OSD.cc:8616 #8 0x00007fbb05a82e9a in OSD::handle_op (this=this@entry=0x7fbb1114a000, op=std::shared_ptr (count 2, weak 0) 0x7fbb1144f800, osdmap=warning: RTTI symbol not found for class 'std::_Sp_counted_deleter, std::hash >::Cleanup, std::allocator, (__gnu_cxx::_Lock_policy)2>' warning: RTTI symbol not found for class 'std::_Sp_counted_deleter, std::hash >::Cleanup, std::allocator, (__gnu_cxx::_Lock_policy)2>' std::shared_ptr (count 31, weak 1) 0x7fbb126fda80) at osd/OSD.cc:8466 #9 0x00007fbb05a840ce in OSD::dispatch_op_fast (this=this@entry=0x7fbb1114a000, op=std::shared_ptr (count 2, weak 0) 0x7fbb1144f800, osdmap=warning: RTTI symbol not found for class 'std::_Sp_counted_deleter, std::hash >::Cleanup, std::allocator, (__gnu_cxx::_Lock_policy)2>' warning: RTTI symbol not found for class 'std::_Sp_counted_deleter, std::hash >::Cleanup, std::allocator, (__gnu_cxx::_Lock_policy)2>' std::shared_ptr (count 31, weak 1) 0x7fbb126fda80) at osd/OSD.cc:6135 #10 0x00007fbb05a843e8 in OSD::dispatch_session_waiting (this=this@entry=0x7fbb1114a000, session=session@entry=0x7fbb1345d260, osdmap=warning: RTTI symbol not found for class 'std::_Sp_counted_deleter, std::hash >::Cleanup, std::allocator, (__gnu_cxx::_Lock_policy)2>' warning: RTTI symbol not found for class 'std::_Sp_counted_deleter, std::hash >::Cleanup, std::allocator, (__gnu_cxx::_Lock_policy)2>' std::shared_ptr (count 31, weak 1) 0x7fbb126fda80) at osd/OSD.cc:5771 #11 0x00007fbb05a84794 in OSD::ms_fast_dispatch (this=0x7fbb1114a000, m=) at osd/OSD.cc:5883 #12 0x00007fbb061c3cf6 in ms_fast_dispatch (m=0x7fbb12214dc0, this=0x7fbb10fa4000) at msg/Messenger.h:555 #13 DispatchQueue::fast_dispatch (this=0x7fbb10fa41d0, m=m@entry=0x7fbb12214dc0) at msg/simple/DispatchQueue.cc:71 #14 0x00007fbb061f6372 in Pipe::DelayedDelivery::entry (this=0x7fbb11733560) at msg/simple/Pipe.cc:226 #15 0x00007fbb03e1cdc5 in start_thread () from /lib64/libpthread.so.0 #16 0x00007fbb024a828d in ?? () from /lib64/libc.so.6 #17 0x0000000000000000 in ?? () Thread 60 (Thread 0x7fbae0b7d700 (LWP 17062)): #0 0x00007fbb0249dc3d in __statvfs_getflags () from /lib64/libc.so.6 #1 0x0000000000000000 in ?? () Thread 59 (Thread 0x7fbae948f700 (LWP 17015)): #0 0x00007fbb03e20a82 in pthread_cond_timedwait@@GLIBC_2.3.2 () from /lib64/libpthread.so.0 #1 0x00007fbb05a7a6e5 in WaitUntil (when=..., mutex=..., this=0x7fbb10edc918) at common/Cond.h:72 #2 WaitInterval (interval=..., mutex=..., cct=, this=0x7fbb10edc918) at common/Cond.h:81 #3 OSD::ShardedOpWQ::_process (this=0x7fbb1114b1e8, thread_index=, hb=0x7fbb111935a0) at osd/OSD.cc:8644 #4 0x00007fbb060f36e7 in ShardedThreadPool::shardedthreadpool_worker (this=0x7fbb1114a768, thread_index=) at common/WorkQueue.cc:340 #5 0x00007fbb060f5650 in ShardedThreadPool::WorkThreadSharded::entry (this=) at common/WorkQueue.h:684 #6 0x00007fbb03e1cdc5 in start_thread () from /lib64/libpthread.so.0 #7 0x00007fbb024a828d in ?? () from /lib64/libc.so.6 #8 0x0000000000000000 in ?? () Thread 58 (Thread 0x7fbaf663f700 (LWP 16984)): #0 0x00007fbb03e206d5 in pthread_cond_wait@@GLIBC_2.3.2 () from /lib64/libpthread.so.0 #1 0x00007fbb06031378 in Wait (mutex=..., this=0x7fbb10f941c0) at common/Cond.h:56 #2 Finisher::finisher_thread_entry (this=0x7fbb10f94160) at common/Finisher.cc:96 #3 0x00007fbb03e1cdc5 in start_thread () from /lib64/libpthread.so.0 #4 0x00007fbb024a828d in ?? () from /lib64/libc.so.6 #5 0x0000000000000000 in ?? () Thread 57 (Thread 0x7fbaecc96700 (LWP 17008)): #0 0x00007fbb0249dc3d in __statvfs_getflags () from /lib64/libc.so.6 #1 0x0000000000000000 in ?? () Thread 56 (Thread 0x7fbad8357700 (LWP 18880)): #0 0x00007fbb03e206d5 in pthread_cond_wait@@GLIBC_2.3.2 () from /lib64/libpthread.so.0 #1 0x00007fbb061f60b9 in Wait (mutex=..., this=0x7fbb11471098) at common/Cond.h:56 #2 Pipe::DelayedDelivery::entry (this=0x7fbb11470fc0) at msg/simple/Pipe.cc:203 #3 0x00007fbb03e1cdc5 in start_thread () from /lib64/libpthread.so.0 #4 0x00007fbb024a828d in ?? () from /lib64/libc.so.6 #5 0x0000000000000000 in ?? () Thread 55 (Thread 0x7fbadad65700 (LWP 17341)): #0 0x00007fbb03e206d5 in pthread_cond_wait@@GLIBC_2.3.2 () from /lib64/libpthread.so.0 #1 0x00007fbb062055cf in Wait (mutex=..., this=0x7fbb112e3248) at common/Cond.h:56 #2 Pipe::writer (this=0x7fbb112e2000) at msg/simple/Pipe.cc:1864 #3 0x00007fbb062084dd in Pipe::Writer::entry (this=) at msg/simple/Pipe.h:63 #4 0x00007fbb03e1cdc5 in start_thread () from /lib64/libpthread.so.0 #5 0x00007fbb024a828d in ?? () from /lib64/libc.so.6 #6 0x0000000000000000 in ?? () Thread 54 (Thread 0x7fbaf7641700 (LWP 16982)): #0 0x00007fbb03e20a82 in pthread_cond_timedwait@@GLIBC_2.3.2 () from /lib64/libpthread.so.0 #1 0x00007fbb060f43da in WaitUntil (when=..., mutex=..., this=0x7fbb10f90b20) at common/Cond.h:72 #2 WaitInterval (interval=..., mutex=..., cct=, this=0x7fbb10f90b20) at common/Cond.h:81 #3 ThreadPool::worker (this=0x7fbb10f90aa0, wt=0x7fbb11156160) at common/WorkQueue.cc:151 #4 0x00007fbb060f5670 in ThreadPool::WorkThread::entry (this=) at common/WorkQueue.h:445 #5 0x00007fbb03e1cdc5 in start_thread () from /lib64/libpthread.so.0 #6 0x00007fbb024a828d in ?? () from /lib64/libc.so.6 #7 0x0000000000000000 in ?? () Thread 53 (Thread 0x7fbae9c90700 (LWP 17014)): #0 0x00007fbb03e20a82 in pthread_cond_timedwait@@GLIBC_2.3.2 () from /lib64/libpthread.so.0 #1 0x00007fbb05a7a6e5 in WaitUntil (when=..., mutex=..., this=0x7fbb10edc7d8) at common/Cond.h:72 #2 WaitInterval (interval=..., mutex=..., cct=, this=0x7fbb10edc7d8) at common/Cond.h:81 #3 OSD::ShardedOpWQ::_process (this=0x7fbb1114b1e8, thread_index=, hb=0x7fbb11193540) at osd/OSD.cc:8644 #4 0x00007fbb060f36e7 in ShardedThreadPool::shardedthreadpool_worker (this=0x7fbb1114a768, thread_index=) at common/WorkQueue.cc:340 #5 0x00007fbb060f5650 in ShardedThreadPool::WorkThreadSharded::entry (this=) at common/WorkQueue.h:684 #6 0x00007fbb03e1cdc5 in start_thread () from /lib64/libpthread.so.0 #7 0x00007fbb024a828d in ?? () from /lib64/libc.so.6 #8 0x0000000000000000 in ?? () Thread 52 (Thread 0x7fbae037c700 (LWP 22007)): #0 0x00007fbb03e206d5 in pthread_cond_wait@@GLIBC_2.3.2 () from /lib64/libpthread.so.0 #1 0x00007fbb062055cf in Wait (mutex=..., this=0x7fbb11297248) at common/Cond.h:56 #2 Pipe::writer (this=0x7fbb11296000) at msg/simple/Pipe.cc:1864 ---Type to continue, or q to quit--- #3 0x00007fbb062084dd in Pipe::Writer::entry (this=) at msg/simple/Pipe.h:63 #4 0x00007fbb03e1cdc5 in start_thread () from /lib64/libpthread.so.0 #5 0x00007fbb024a828d in ?? () from /lib64/libc.so.6 #6 0x0000000000000000 in ?? () Thread 51 (Thread 0x7fbaee499700 (LWP 17005)): #0 0x00007fbb0249dc3d in __statvfs_getflags () from /lib64/libc.so.6 #1 0x0000000000000000 in ?? () Thread 50 (Thread 0x7fbadb566700 (LWP 17336)): #0 0x00007fbb03e206d5 in pthread_cond_wait@@GLIBC_2.3.2 () from /lib64/libpthread.so.0 #1 0x00007fbb061f60b9 in Wait (mutex=..., this=0x7fbb1104ba38) at common/Cond.h:56 #2 Pipe::DelayedDelivery::entry (this=0x7fbb1104b960) at msg/simple/Pipe.cc:203 #3 0x00007fbb03e1cdc5 in start_thread () from /lib64/libpthread.so.0 #4 0x00007fbb024a828d in ?? () from /lib64/libc.so.6 #5 0x0000000000000000 in ?? () Thread 49 (Thread 0x7fbae0c7e700 (LWP 17061)): #0 0x00007fbb0249dc3d in __statvfs_getflags () from /lib64/libc.so.6 #1 0x0000000000000000 in ?? () Thread 48 (Thread 0x7fbade573700 (LWP 21073)): #0 0x00007fbb03e206d5 in pthread_cond_wait@@GLIBC_2.3.2 () from /lib64/libpthread.so.0 #1 0x00007fbb061ff0ad in Wait (mutex=..., this=0x7fbb11288648) at common/Cond.h:56 #2 Pipe::reader (this=0x7fbb11287400) at msg/simple/Pipe.cc:1529 #3 0x00007fbb062084bd in Pipe::Reader::entry (this=) at msg/simple/Pipe.h:52 #4 0x00007fbb03e1cdc5 in start_thread () from /lib64/libpthread.so.0 #5 0x00007fbb024a828d in ?? () from /lib64/libc.so.6 #6 0x0000000000000000 in ?? () Thread 47 (Thread 0x7fbaeb493700 (LWP 17011)): #0 0x00007fbb03e20a82 in pthread_cond_timedwait@@GLIBC_2.3.2 () from /lib64/libpthread.so.0 #1 0x00007fbb060f43da in WaitUntil (when=..., mutex=..., this=0x7fbb1114a650) at common/Cond.h:72 #2 WaitInterval (interval=..., mutex=..., cct=, this=0x7fbb1114a650) at common/Cond.h:81 #3 ThreadPool::worker (this=0x7fbb1114a5d0, wt=0x7fbb11192760) at common/WorkQueue.cc:151 #4 0x00007fbb060f5670 in ThreadPool::WorkThread::entry (this=) at common/WorkQueue.h:445 #5 0x00007fbb03e1cdc5 in start_thread () from /lib64/libpthread.so.0 #6 0x00007fbb024a828d in ?? () from /lib64/libc.so.6 #7 0x0000000000000000 in ?? () Thread 46 (Thread 0x7fbadda6f700 (LWP 21308)): #0 0x00007fbb03e22f4d in __lll_lock_wait () from /lib64/libpthread.so.0 #1 0x00007fbb03e254bc in _L_cond_lock_792 () from /lib64/libpthread.so.0 #2 0x00007fbb03e25398 in __pthread_mutex_cond_lock () from /lib64/libpthread.so.0 #3 0x00007fbb03e20765 in pthread_cond_wait@@GLIBC_2.3.2 () from /lib64/libpthread.so.0 #4 0x00007fbb062055cf in Wait (mutex=..., this=0x7fbb13d0d248) at common/Cond.h:56 #5 Pipe::writer (this=0x7fbb13d0c000) at msg/simple/Pipe.cc:1864 #6 0x00007fbb062084dd in Pipe::Writer::entry (this=) at msg/simple/Pipe.h:63 #7 0x00007fbb03e1cdc5 in start_thread () from /lib64/libpthread.so.0 #8 0x00007fbb024a828d in ?? () from /lib64/libc.so.6 #9 0x0000000000000000 in ?? () Thread 45 (Thread 0x7fbadab63700 (LWP 21310)): #0 0x00007fbb03e206d5 in pthread_cond_wait@@GLIBC_2.3.2 () from /lib64/libpthread.so.0 #1 0x00007fbb061ed78f in Wait (mutex=..., this=0x7fbb11733638) at common/Cond.h:56 #2 Pipe::DelayedDelivery::stop_fast_dispatching (this=0x7fbb11733560) at msg/simple/Pipe.cc:249 #3 0x00007fbb061ee99f in Pipe::stop_and_wait (this=this@entry=0x7fbb13d0c000) at msg/simple/Pipe.cc:1502 #4 0x00007fbb061fcd3f in Pipe::accept (this=this@entry=0x7fbb117be800) at msg/simple/Pipe.cc:696 #5 0x00007fbb062007cf in Pipe::reader (this=0x7fbb117be800) at msg/simple/Pipe.cc:1517 #6 0x00007fbb062084bd in Pipe::Reader::entry (this=) at msg/simple/Pipe.h:52 #7 0x00007fbb03e1cdc5 in start_thread () from /lib64/libpthread.so.0 #8 0x00007fbb024a828d in ?? () from /lib64/libc.so.6 #9 0x0000000000000000 in ?? () Thread 44 (Thread 0x7fbae1c80700 (LWP 17030)): #0 0x00007fbb03e206d5 in pthread_cond_wait@@GLIBC_2.3.2 () from /lib64/libpthread.so.0 #1 0x00007fbb060ec6ff in Wait (mutex=..., this=0x7fbb1114c1b0) at common/Cond.h:56 #2 SafeTimer::timer_thread (this=0x7fbb1114c1a0) at common/Timer.cc:116 #3 0x00007fbb060edd3d in SafeTimerThread::entry (this=) at common/Timer.cc:38 #4 0x00007fbb03e1cdc5 in start_thread () from /lib64/libpthread.so.0 #5 0x00007fbb024a828d in ?? () from /lib64/libc.so.6 #6 0x0000000000000000 in ?? () Thread 43 (Thread 0x7fbaef49b700 (LWP 17003)): #0 0x00007fbb03e22f4d in __lll_lock_wait () from /lib64/libpthread.so.0 #1 0x00007fbb03e1ed02 in _L_lock_791 () from /lib64/libpthread.so.0 #2 0x00007fbb03e1ec08 in pthread_mutex_lock () from /lib64/libpthread.so.0 #3 0x00007fbb060b47d8 in Mutex::Lock (this=this@entry=0x7ffd68175960, no_lockdep=no_lockdep@entry=false) at common/Mutex.cc:110 #4 0x00007fbb05a63e58 in Locker (m=..., this=) at common/Mutex.h:115 #5 sub_want_increment (flags=1, start=35, what="\230vR\023\273\177\000\000\000\341\335\177\206\261\006\026ЍI\357\272\177\000\000\000\240\024\021\273\177\000\000\300\215I\357\272\177\000\000\030\263\024\021\273\177\000\000#\000\000\000\000\000\000\000ЍI\357\272\177\000\000\200%\367\022\273\177\000\000\254@\246\005\273\177\000\000#\000\000\000\000\000\000\000۪\004\006\273\177\000\000؍I\357\272\177\000\000\000\341\335\177\206\261\006\026\000\240\024\021\273\177\000\000@\225I\357\272\177\000\000\"\000\000\000\000\000\000\000\277A\246\005\273\177\000\000\200\212\031\021\273\177\000\000@\301\355\020\273\177\000\000@\301\355\020\273\177\000\000\312V\321\002\"\000\000\000\000(H\021\273\177\000\000\340u\330\022\273\177\000\000\370\216\367\002\273\177\000\000\060o\367\002"...
, this=0x7ffd681757d0) at mon/MonClient.h:280 #6 OSD::osdmap_subscribe (this=this@entry=0x7fbb1114a000, epoch=35, force_request=force_request@entry=false) at osd/OSD.cc:6535 #7 0x00007fbb05a640ac in OSD::wait_for_new_map (this=this@entry=0x7fbb1114a000, op=std::shared_ptr (count 4, weak 0) 0x7fbb11482800) at osd/OSD.cc:6468 #8 0x00007fbb05a641bf in OSD::require_same_or_newer_map (this=this@entry=0x7fbb1114a000, op=std::shared_ptr (count 4, weak 0) 0x7fbb11482800, epoch=35, is_fast_dispatch=is_fast_dispatch@entry=false) at osd/OSD.cc:7370 ---Type to continue, or q to quit--- #9 0x00007fbb05a9a19c in OSD::handle_pg_notify (this=0x7fbb1114a000, op=std::shared_ptr (count 4, weak 0) 0x7fbb11482800) at osd/OSD.cc:7766 #10 0x00007fbb05a9b730 in OSD::dispatch_op (this=this@entry=0x7fbb1114a000, op=std::shared_ptr (count 4, weak 0) 0x7fbb11482800) at osd/OSD.cc:6078 #11 0x00007fbb05a9c35e in OSD::_dispatch (this=this@entry=0x7fbb1114a000, m=m@entry=0x7fbb12f72580) at osd/OSD.cc:6252 #12 0x00007fbb05a9ca1f in OSD::ms_dispatch (this=0x7fbb1114a000, m=0x7fbb12f72580) at osd/OSD.cc:5755 #13 0x00007fbb061c5b9a in ms_deliver_dispatch (m=0x7fbb12f72580, this=0x7fbb10fa4700) at msg/Messenger.h:584 #14 DispatchQueue::entry (this=0x7fbb10fa48d0) at msg/simple/DispatchQueue.cc:185 #15 0x00007fbb060def3d in DispatchQueue::DispatchThread::entry (this=) at msg/simple/DispatchQueue.h:103 #16 0x00007fbb03e1cdc5 in start_thread () from /lib64/libpthread.so.0 #17 0x00007fbb024a828d in ?? () from /lib64/libc.so.6 #18 0x0000000000000000 in ?? () Thread 42 (Thread 0x7fbadef76700 (LWP 17269)): #0 0x00007fbb03e22f4d in __lll_lock_wait () from /lib64/libpthread.so.0 #1 0x00007fbb03e1ed02 in _L_lock_791 () from /lib64/libpthread.so.0 #2 0x00007fbb03e1ec08 in pthread_mutex_lock () from /lib64/libpthread.so.0 #3 0x00007fbb060b47d8 in Mutex::Lock (this=this@entry=0x7ffd68175960, no_lockdep=no_lockdep@entry=false) at common/Mutex.cc:110 #4 0x00007fbb05a63e58 in Locker (m=..., this=) at common/Mutex.h:115 #5 sub_want_increment (flags=1, start=35, what="\230~\330\022\273\177\000\000\000\341\335\177\206\261\006\026\000\240\024\021\273\177\000\000@\301\355\020\273\177\000\000@Z\376\021\273\177\000\000\360C\367\336\272\177\000\000\000\240\024\021\273\177\000\000\060\256\024\021\273\177\000\000\240Fk\021\273\177\000\000\352s\247\005\273\177\000\000\200\332o\022\273\177\000\000\000\240\355\022\273\177\000\000\220D\367\336\272\177\000\000\060\000\000\000\273\177\000\000\314C\367\336\272\177\000\000\200|/\022\273\177\000\000P\200#\021\273\177\000\000\334Ӵ\021\273\177\000\000X\200#\021\273\177\000\000H\200#\021\273\177\000\000@\200#\021\273\177\000\000\312V\321\002\002", '\000' , "\002\000\000\000\000\204.\021\273\177\000\000\242U\321\002"...
, this=0x7ffd681757d0) at mon/MonClient.h:280 #6 OSD::osdmap_subscribe (this=this@entry=0x7fbb1114a000, epoch=35, force_request=force_request@entry=false) at osd/OSD.cc:6535 #7 0x00007fbb05a773ea in OSD::handle_osd_ping (this=this@entry=0x7fbb1114a000, m=m@entry=0x7fbb12eda000) at osd/OSD.cc:4001 #8 0x00007fbb05a7866b in OSD::heartbeat_dispatch (this=0x7fbb1114a000, m=0x7fbb12eda000) at osd/OSD.cc:5713 #9 0x00007fbb06213d9b in ms_fast_dispatch (m=, this=0x7fbb10fac000) at msg/Messenger.h:555 #10 AsyncConnection::DelayedDelivery::do_request (this=, id=) at msg/async/AsyncConnection.cc:2453 #11 0x00007fbb061c8626 in EventCenter::process_time_events (this=this@entry=0x7fbb10faa848) at msg/async/Event.cc:307 #12 0x00007fbb061c9d78 in EventCenter::process_events (this=this@entry=0x7fbb10faa848, timeout_microseconds=, timeout_microseconds@entry=30000000) at msg/async/Event.cc:392 #13 0x00007fbb061a90e0 in Worker::entry (this=0x7fbb10faa800) at msg/async/AsyncMessenger.cc:294 #14 0x00007fbb03e1cdc5 in start_thread () from /lib64/libpthread.so.0 #15 0x00007fbb024a828d in ?? () from /lib64/libc.so.6 #16 0x0000000000000000 in ?? () Thread 41 (Thread 0x7fbaf3e3a700 (LWP 16989)): #0 0x00007fbb03e206d5 in pthread_cond_wait@@GLIBC_2.3.2 () from /lib64/libpthread.so.0 #1 0x00007fbb06031378 in Wait (mutex=..., this=0x7fbb10f948a0) at common/Cond.h:56 #2 Finisher::finisher_thread_entry (this=0x7fbb10f94840) at common/Finisher.cc:96 #3 0x00007fbb03e1cdc5 in start_thread () from /lib64/libpthread.so.0 #4 0x00007fbb024a828d in ?? () from /lib64/libc.so.6 #5 0x0000000000000000 in ?? () Thread 40 (Thread 0x7fbae147f700 (LWP 17031)): #0 0x00007fbb03e206d5 in pthread_cond_wait@@GLIBC_2.3.2 () from /lib64/libpthread.so.0 #1 0x00007fbb05aac57e in Cond::Wait (this=0x7fbb1114c050, mutex=...) at common/Cond.h:56 #2 0x00007fbb05a4412c in OSDService::agent_entry (this=0x7fbb1114b860) at osd/OSD.cc:552 #3 0x00007fbb05ab242d in OSDService::AgentThread::entry (this=) at osd/OSD.h:676 #4 0x00007fbb03e1cdc5 in start_thread () from /lib64/libpthread.so.0 #5 0x00007fbb024a828d in ?? () from /lib64/libc.so.6 #6 0x0000000000000000 in ?? () Thread 39 (Thread 0x7fbadaa62700 (LWP 17353)): #0 0x00007fbb03e206d5 in pthread_cond_wait@@GLIBC_2.3.2 () from /lib64/libpthread.so.0 #1 0x00007fbb061ff0ad in Wait (mutex=..., this=0x7fbb112e3248) at common/Cond.h:56 #2 Pipe::reader (this=0x7fbb112e2000) at msg/simple/Pipe.cc:1529 #3 0x00007fbb062084bd in Pipe::Reader::entry (this=) at msg/simple/Pipe.h:52 #4 0x00007fbb03e1cdc5 in start_thread () from /lib64/libpthread.so.0 #5 0x00007fbb024a828d in ?? () from /lib64/libc.so.6 #6 0x0000000000000000 in ?? () Thread 38 (Thread 0x7fbae6489700 (LWP 17021)): #0 0x00007fbb03e20a82 in pthread_cond_timedwait@@GLIBC_2.3.2 () from /lib64/libpthread.so.0 #1 0x00007fbb05a7a6e5 in WaitUntil (when=..., mutex=..., this=0x7fbb10edca58) at common/Cond.h:72 #2 WaitInterval (interval=..., mutex=..., cct=, this=0x7fbb10edca58) at common/Cond.h:81 #3 OSD::ShardedOpWQ::_process (this=0x7fbb1114b1e8, thread_index=, hb=0x7fbb111936c0) at osd/OSD.cc:8644 #4 0x00007fbb060f36e7 in ShardedThreadPool::shardedthreadpool_worker (this=0x7fbb1114a768, thread_index=) at common/WorkQueue.cc:340 #5 0x00007fbb060f5650 in ShardedThreadPool::WorkThreadSharded::entry (this=) at common/WorkQueue.h:684 #6 0x00007fbb03e1cdc5 in start_thread () from /lib64/libpthread.so.0 #7 0x00007fbb024a828d in ?? () from /lib64/libc.so.6 #8 0x0000000000000000 in ?? () Thread 37 (Thread 0x7fbae2481700 (LWP 17029)): #0 0x00007fbb03e206d5 in pthread_cond_wait@@GLIBC_2.3.2 () from /lib64/libpthread.so.0 #1 0x00007fbb060ec6ff in Wait (mutex=..., this=0x7fbb1114c460) at common/Cond.h:56 #2 SafeTimer::timer_thread (this=0x7fbb1114c450) at common/Timer.cc:116 #3 0x00007fbb060edd3d in SafeTimerThread::entry (this=) at common/Timer.cc:38 #4 0x00007fbb03e1cdc5 in start_thread () from /lib64/libpthread.so.0 #5 0x00007fbb024a828d in ?? () from /lib64/libc.so.6 #6 0x0000000000000000 in ?? () Thread 36 (Thread 0x7fbaf463b700 (LWP 16988)): #0 0x00007fbb03e206d5 in pthread_cond_wait@@GLIBC_2.3.2 () from /lib64/libpthread.so.0 #1 0x00007fbb06031378 in Wait (mutex=..., this=0x7fbb10f94740) at common/Cond.h:56 #2 Finisher::finisher_thread_entry (this=0x7fbb10f946e0) at common/Finisher.cc:96 #3 0x00007fbb03e1cdc5 in start_thread () from /lib64/libpthread.so.0 #4 0x00007fbb024a828d in ?? () from /lib64/libc.so.6 #5 0x0000000000000000 in ?? () ---Type to continue, or q to quit--- Thread 35 (Thread 0x7fbae5c88700 (LWP 17022)): #0 0x00007fbb03e20a82 in pthread_cond_timedwait@@GLIBC_2.3.2 () from /lib64/libpthread.so.0 #1 0x00007fbb05a7a6e5 in WaitUntil (when=..., mutex=..., this=0x7fbb10edcb98) at common/Cond.h:72 #2 WaitInterval (interval=..., mutex=..., cct=, this=0x7fbb10edcb98) at common/Cond.h:81 #3 OSD::ShardedOpWQ::_process (this=0x7fbb1114b1e8, thread_index=, hb=0x7fbb111936f0) at osd/OSD.cc:8644 #4 0x00007fbb060f36e7 in ShardedThreadPool::shardedthreadpool_worker (this=0x7fbb1114a768, thread_index=) at common/WorkQueue.cc:340 #5 0x00007fbb060f5650 in ShardedThreadPool::WorkThreadSharded::entry (this=) at common/WorkQueue.h:684 #6 0x00007fbb03e1cdc5 in start_thread () from /lib64/libpthread.so.0 #7 0x00007fbb024a828d in ?? () from /lib64/libc.so.6 #8 0x0000000000000000 in ?? () Thread 34 (Thread 0x7fbaf6e40700 (LWP 16983)): #0 0x00007fbb03e20a82 in pthread_cond_timedwait@@GLIBC_2.3.2 () from /lib64/libpthread.so.0 #1 0x00007fbb060f43da in WaitUntil (when=..., mutex=..., this=0x7fbb10f90b20) at common/Cond.h:72 #2 WaitInterval (interval=..., mutex=..., cct=, this=0x7fbb10f90b20) at common/Cond.h:81 #3 ThreadPool::worker (this=0x7fbb10f90aa0, wt=0x7fbb111561c0) at common/WorkQueue.cc:151 #4 0x00007fbb060f5670 in ThreadPool::WorkThread::entry (this=) at common/WorkQueue.h:445 #5 0x00007fbb03e1cdc5 in start_thread () from /lib64/libpthread.so.0 #6 0x00007fbb024a828d in ?? () from /lib64/libc.so.6 #7 0x0000000000000000 in ?? () Thread 33 (Thread 0x7fbae2c82700 (LWP 17028)): #0 0x00007fbb03e206d5 in pthread_cond_wait@@GLIBC_2.3.2 () from /lib64/libpthread.so.0 #1 0x00007fbb06031378 in Wait (mutex=..., this=0x7fbb1114c308) at common/Cond.h:56 #2 Finisher::finisher_thread_entry (this=0x7fbb1114c2a8) at common/Finisher.cc:96 #3 0x00007fbb03e1cdc5 in start_thread () from /lib64/libpthread.so.0 #4 0x00007fbb024a828d in ?? () from /lib64/libc.so.6 #5 0x0000000000000000 in ?? () Thread 32 (Thread 0x7fbae3483700 (LWP 17027)): #0 0x00007fbb03e206d5 in pthread_cond_wait@@GLIBC_2.3.2 () from /lib64/libpthread.so.0 #1 0x00007fbb06031378 in Wait (mutex=..., this=0x7fbb1114c6a8) at common/Cond.h:56 #2 Finisher::finisher_thread_entry (this=0x7fbb1114c648) at common/Finisher.cc:96 #3 0x00007fbb03e1cdc5 in start_thread () from /lib64/libpthread.so.0 #4 0x00007fbb024a828d in ?? () from /lib64/libc.so.6 #5 0x0000000000000000 in ?? () Thread 31 (Thread 0x7fbae848d700 (LWP 17017)): #0 0x00007fbb03e20a82 in pthread_cond_timedwait@@GLIBC_2.3.2 () from /lib64/libpthread.so.0 #1 0x00007fbb05a7a6e5 in WaitUntil (when=..., mutex=..., this=0x7fbb10edcb98) at common/Cond.h:72 #2 WaitInterval (interval=..., mutex=..., cct=, this=0x7fbb10edcb98) at common/Cond.h:81 #3 OSD::ShardedOpWQ::_process (this=0x7fbb1114b1e8, thread_index=, hb=0x7fbb11193600) at osd/OSD.cc:8644 #4 0x00007fbb060f36e7 in ShardedThreadPool::shardedthreadpool_worker (this=0x7fbb1114a768, thread_index=) at common/WorkQueue.cc:340 #5 0x00007fbb060f5650 in ShardedThreadPool::WorkThreadSharded::entry (this=) at common/WorkQueue.h:684 #6 0x00007fbb03e1cdc5 in start_thread () from /lib64/libpthread.so.0 #7 0x00007fbb024a828d in ?? () from /lib64/libc.so.6 #8 0x0000000000000000 in ?? () Thread 30 (Thread 0x7fbaf4e3c700 (LWP 16987)): #0 0x00007fbb03e206d5 in pthread_cond_wait@@GLIBC_2.3.2 () from /lib64/libpthread.so.0 #1 0x00007fbb06031378 in Wait (mutex=..., this=0x7fbb10f945e0) at common/Cond.h:56 #2 Finisher::finisher_thread_entry (this=0x7fbb10f94580) at common/Finisher.cc:96 #3 0x00007fbb03e1cdc5 in start_thread () from /lib64/libpthread.so.0 #4 0x00007fbb024a828d in ?? () from /lib64/libc.so.6 #5 0x0000000000000000 in ?? () Thread 29 (Thread 0x7fbae6c8a700 (LWP 17020)): #0 0x00007fbb03e20a82 in pthread_cond_timedwait@@GLIBC_2.3.2 () from /lib64/libpthread.so.0 #1 0x00007fbb05a7a6e5 in WaitUntil (when=..., mutex=..., this=0x7fbb10edc918) at common/Cond.h:72 #2 WaitInterval (interval=..., mutex=..., cct=, this=0x7fbb10edc918) at common/Cond.h:81 #3 OSD::ShardedOpWQ::_process (this=0x7fbb1114b1e8, thread_index=, hb=0x7fbb11193690) at osd/OSD.cc:8644 #4 0x00007fbb060f36e7 in ShardedThreadPool::shardedthreadpool_worker (this=0x7fbb1114a768, thread_index=) at common/WorkQueue.cc:340 #5 0x00007fbb060f5650 in ShardedThreadPool::WorkThreadSharded::entry (this=) at common/WorkQueue.h:684 #6 0x00007fbb03e1cdc5 in start_thread () from /lib64/libpthread.so.0 #7 0x00007fbb024a828d in ?? () from /lib64/libc.so.6 #8 0x0000000000000000 in ?? () Thread 28 (Thread 0x7fbaea491700 (LWP 17013)): #0 0x00007fbb03e22f4d in __lll_lock_wait () from /lib64/libpthread.so.0 #1 0x00007fbb03e1ed02 in _L_lock_791 () from /lib64/libpthread.so.0 #2 0x00007fbb03e1ec08 in pthread_mutex_lock () from /lib64/libpthread.so.0 #3 0x00007fbb060b47d8 in Mutex::Lock (this=this@entry=0x7fbb1142d528, no_lockdep=no_lockdep@entry=false) at common/Mutex.cc:110 #4 0x00007fbb05b0df83 in PG::lock (this=this@entry=0x7fbb1142d000, no_lockdep=no_lockdep@entry=false) at osd/PG.cc:265 #5 0x00007fbb05b0e11e in PG::lock_suspend_timeout (this=0x7fbb1142d000, handle=...) at osd/PG.cc:259 #6 0x00007fbb05a7a869 in OSD::ShardedOpWQ::_process (this=0x7fbb1114b1e8, thread_index=, hb=0x7fbb11193510) at osd/OSD.cc:8658 #7 0x00007fbb060f36e7 in ShardedThreadPool::shardedthreadpool_worker (this=0x7fbb1114a768, thread_index=) at common/WorkQueue.cc:340 #8 0x00007fbb060f5650 in ShardedThreadPool::WorkThreadSharded::entry (this=) at common/WorkQueue.h:684 #9 0x00007fbb03e1cdc5 in start_thread () from /lib64/libpthread.so.0 #10 0x00007fbb024a828d in ?? () from /lib64/libc.so.6 #11 0x0000000000000000 in ?? () Thread 27 (Thread 0x7fbaf7e42700 (LWP 16981)): #0 0x00007fbb03e206d5 in pthread_cond_wait@@GLIBC_2.3.2 () from /lib64/libpthread.so.0 #1 0x00007fbb06031378 in Wait (mutex=..., this=0x7fbb10f90080) at common/Cond.h:56 #2 Finisher::finisher_thread_entry (this=0x7fbb10f90020) at common/Finisher.cc:96 ---Type to continue, or q to quit--- #3 0x00007fbb03e1cdc5 in start_thread () from /lib64/libpthread.so.0 #4 0x00007fbb024a828d in ?? () from /lib64/libc.so.6 #5 0x0000000000000000 in ?? () Thread 26 (Thread 0x7fbaebc94700 (LWP 17010)): #0 0x00007fbb03e206d5 in pthread_cond_wait@@GLIBC_2.3.2 () from /lib64/libpthread.so.0 #1 0x00007fbb06031378 in Wait (mutex=..., this=0x7ffd68175ae0) at common/Cond.h:56 #2 Finisher::finisher_thread_entry (this=0x7ffd68175a80) at common/Finisher.cc:96 #3 0x00007fbb03e1cdc5 in start_thread () from /lib64/libpthread.so.0 #4 0x00007fbb024a828d in ?? () from /lib64/libc.so.6 #5 0x0000000000000000 in ?? () Thread 25 (Thread 0x7fbae3c84700 (LWP 17026)): #0 0x00007fbb03e22f4d in __lll_lock_wait () from /lib64/libpthread.so.0 #1 0x00007fbb03e254bc in _L_cond_lock_792 () from /lib64/libpthread.so.0 #2 0x00007fbb03e25398 in __pthread_mutex_cond_lock () from /lib64/libpthread.so.0 #3 0x00007fbb03e20b54 in pthread_cond_timedwait@@GLIBC_2.3.2 () from /lib64/libpthread.so.0 #4 0x00007fbb05a6df4d in WaitUntil (when=..., mutex=..., this=0x7fbb1114aeb8) at common/Cond.h:72 #5 WaitInterval (interval=..., mutex=..., cct=, this=0x7fbb1114aeb8) at common/Cond.h:81 #6 OSD::heartbeat_entry (this=0x7fbb1114a000) at osd/OSD.cc:4021 #7 0x00007fbb05ad941d in OSD::T_Heartbeat::entry (this=) at osd/OSD.h:1590 #8 0x00007fbb03e1cdc5 in start_thread () from /lib64/libpthread.so.0 #9 0x00007fbb024a828d in ?? () from /lib64/libc.so.6 #10 0x0000000000000000 in ?? () Thread 24 (Thread 0x7fbaedc98700 (LWP 17006)): #0 0x00007fbb03e206d5 in pthread_cond_wait@@GLIBC_2.3.2 () from /lib64/libpthread.so.0 #1 0x00007fbb061c56db in Wait (mutex=..., this=0x7fbb10fa5038) at common/Cond.h:56 #2 DispatchQueue::entry (this=0x7fbb10fa4fd0) at msg/simple/DispatchQueue.cc:196 #3 0x00007fbb060def3d in DispatchQueue::DispatchThread::entry (this=) at msg/simple/DispatchQueue.h:103 #4 0x00007fbb03e1cdc5 in start_thread () from /lib64/libpthread.so.0 #5 0x00007fbb024a828d in ?? () from /lib64/libc.so.6 #6 0x0000000000000000 in ?? () Thread 23 (Thread 0x7fbaf9645700 (LWP 16957)): #0 0x00007fbb03e206d5 in pthread_cond_wait@@GLIBC_2.3.2 () from /lib64/libpthread.so.0 #1 0x00007fbb05e065af in Wait (mutex=..., this=0x7fbb10f90820) at common/Cond.h:56 #2 WBThrottle::get_next_should_flush (this=this@entry=0x7fbb10f906f8, next=next@entry=0x7fbaf9643d30) at os/filestore/WBThrottle.cc:139 #3 0x00007fbb05e06c32 in WBThrottle::entry (this=0x7fbb10f906f8) at os/filestore/WBThrottle.cc:157 #4 0x00007fbb03e1cdc5 in start_thread () from /lib64/libpthread.so.0 #5 0x00007fbb024a828d in ?? () from /lib64/libc.so.6 #6 0x0000000000000000 in ?? () Thread 22 (Thread 0x7fbaf563d700 (LWP 16986)): #0 0x00007fbb03e206d5 in pthread_cond_wait@@GLIBC_2.3.2 () from /lib64/libpthread.so.0 #1 0x00007fbb06031378 in Wait (mutex=..., this=0x7fbb10f94480) at common/Cond.h:56 #2 Finisher::finisher_thread_entry (this=0x7fbb10f94420) at common/Finisher.cc:96 #3 0x00007fbb03e1cdc5 in start_thread () from /lib64/libpthread.so.0 #4 0x00007fbb024a828d in ?? () from /lib64/libc.so.6 #5 0x0000000000000000 in ?? () Thread 21 (Thread 0x7fbaeac92700 (LWP 17012)): #0 0x00007fbb03e20a82 in pthread_cond_timedwait@@GLIBC_2.3.2 () from /lib64/libpthread.so.0 #1 0x00007fbb060f43da in WaitUntil (when=..., mutex=..., this=0x7fbb1114a650) at common/Cond.h:72 #2 WaitInterval (interval=..., mutex=..., cct=, this=0x7fbb1114a650) at common/Cond.h:81 #3 ThreadPool::worker (this=0x7fbb1114a5d0, wt=0x7fbb11192700) at common/WorkQueue.cc:151 #4 0x00007fbb060f5670 in ThreadPool::WorkThread::entry (this=) at common/WorkQueue.h:445 #5 0x00007fbb03e1cdc5 in start_thread () from /lib64/libpthread.so.0 #6 0x00007fbb024a828d in ?? () from /lib64/libc.so.6 #7 0x0000000000000000 in ?? () Thread 20 (Thread 0x7fbaf8643700 (LWP 16980)): #0 0x00007fbb03e206d5 in pthread_cond_wait@@GLIBC_2.3.2 () from /lib64/libpthread.so.0 #1 0x00007fbb05aac57e in Cond::Wait (this=0x7fbb11158740, mutex=...) at common/Cond.h:56 #2 0x00007fbb05eac3d4 in FileJournal::write_thread_entry (this=0x7fbb11158600) at os/filestore/FileJournal.cc:1273 #3 0x00007fbb05de754d in FileJournal::Writer::entry (this=) at os/filestore/FileJournal.h:372 #4 0x00007fbb03e1cdc5 in start_thread () from /lib64/libpthread.so.0 #5 0x00007fbb024a828d in ?? () from /lib64/libc.so.6 #6 0x0000000000000000 in ?? () Thread 19 (Thread 0x7fbaeec9a700 (LWP 17004)): #0 0x00007fbb03e206d5 in pthread_cond_wait@@GLIBC_2.3.2 () from /lib64/libpthread.so.0 #1 0x00007fbb061c535f in Wait (mutex=..., this=0x7fbb10fa4ae8) at common/Cond.h:56 #2 DispatchQueue::run_local_delivery (this=0x7fbb10fa48d0) at msg/simple/DispatchQueue.cc:114 #3 0x00007fbb060def5d in DispatchQueue::LocalDeliveryThread::entry (this=) at msg/simple/DispatchQueue.h:117 #4 0x00007fbb03e1cdc5 in start_thread () from /lib64/libpthread.so.0 #5 0x00007fbb024a828d in ?? () from /lib64/libc.so.6 #6 0x0000000000000000 in ?? () Thread 18 (Thread 0x7fbaf3639700 (LWP 16990)): #0 0x00007fbb03e20a82 in pthread_cond_timedwait@@GLIBC_2.3.2 () from /lib64/libpthread.so.0 #1 0x00007fbb060ec99e in WaitUntil (when=..., mutex=..., this=0x7fbb10f905d8) at common/Cond.h:72 #2 SafeTimer::timer_thread (this=0x7fbb10f905c8) at common/Timer.cc:118 #3 0x00007fbb060edd3d in SafeTimerThread::entry (this=) at common/Timer.cc:38 #4 0x00007fbb03e1cdc5 in start_thread () from /lib64/libpthread.so.0 #5 0x00007fbb024a828d in ?? () from /lib64/libc.so.6 ---Type to continue, or q to quit--- #6 0x0000000000000000 in ?? () Thread 17 (Thread 0x7fbaec495700 (LWP 17009)): #0 0x00007fbb03e22f4d in __lll_lock_wait () from /lib64/libpthread.so.0 #1 0x00007fbb03e254bc in _L_cond_lock_792 () from /lib64/libpthread.so.0 #2 0x00007fbb03e25398 in __pthread_mutex_cond_lock () from /lib64/libpthread.so.0 #3 0x00007fbb03e20b54 in pthread_cond_timedwait@@GLIBC_2.3.2 () from /lib64/libpthread.so.0 #4 0x00007fbb060ec99e in WaitUntil (when=..., mutex=..., this=0x7ffd681759c8) at common/Cond.h:72 #5 SafeTimer::timer_thread (this=0x7ffd681759b8) at common/Timer.cc:118 #6 0x00007fbb060edd3d in SafeTimerThread::entry (this=) at common/Timer.cc:38 #7 0x00007fbb03e1cdc5 in start_thread () from /lib64/libpthread.so.0 #8 0x00007fbb024a828d in ?? () from /lib64/libc.so.6 #9 0x0000000000000000 in ?? () Thread 16 (Thread 0x7fbafa647700 (LWP 16951)): #0 0x00007fbb03e22f4d in __lll_lock_wait () from /lib64/libpthread.so.0 #1 0x00007fbb03e1ed02 in _L_lock_791 () from /lib64/libpthread.so.0 #2 0x00007fbb03e1ec08 in pthread_mutex_lock () from /lib64/libpthread.so.0 #3 0x00007fbb060b47d8 in Mutex::Lock (this=this@entry=0x7fbb10fa44b0, no_lockdep=no_lockdep@entry=false) at common/Mutex.cc:110 #4 0x00007fbb060d8b04 in SimpleMessenger::mark_down (this=0x7fbb10fa4000, con=0x7fbb10f4a880) at msg/simple/SimpleMessenger.cc:655 #5 0x00007fbb060a4e4f in MonClient::_reopen_session (this=this@entry=0x7ffd681757d0, rank=rank@entry=-1, name="") at mon/MonClient.cc:617 #6 0x00007fbb05a6f9bb in _reopen_session (this=0x7ffd681757d0) at mon/MonClient.h:191 #7 reopen_session (cb=0x0, this=0x7ffd681757d0) at mon/MonClient.h:346 #8 OSD::tick_without_osd_lock (this=0x7fbb1114a000) at osd/OSD.cc:4298 #9 0x00007fbb05aa3e29 in Context::complete (this=0x7fbb10eac3b0, r=) at include/Context.h:64 #10 0x00007fbb060ec304 in SafeTimer::timer_thread (this=0x7fbb1114a190) at common/Timer.cc:105 #11 0x00007fbb060edd3d in SafeTimerThread::entry (this=) at common/Timer.cc:38 #12 0x00007fbb03e1cdc5 in start_thread () from /lib64/libpthread.so.0 #13 0x00007fbb024a828d in ?? () from /lib64/libc.so.6 #14 0x0000000000000000 in ?? () Thread 15 (Thread 0x7fbaf5e3e700 (LWP 16985)): #0 0x00007fbb03e206d5 in pthread_cond_wait@@GLIBC_2.3.2 () from /lib64/libpthread.so.0 #1 0x00007fbb06031378 in Wait (mutex=..., this=0x7fbb10f94320) at common/Cond.h:56 #2 Finisher::finisher_thread_entry (this=0x7fbb10f942c0) at common/Finisher.cc:96 #3 0x00007fbb03e1cdc5 in start_thread () from /lib64/libpthread.so.0 #4 0x00007fbb024a828d in ?? () from /lib64/libc.so.6 #5 0x0000000000000000 in ?? () Thread 14 (Thread 0x7fbaf8e44700 (LWP 16958)): #0 0x00007fbb03e20a82 in pthread_cond_timedwait@@GLIBC_2.3.2 () from /lib64/libpthread.so.0 #1 0x00007fbb05de858d in WaitUntil (when=..., mutex=..., this=0x7fbb10f90530) at common/Cond.h:72 #2 Cond::WaitInterval (this=this@entry=0x7fbb10f90530, cct=, mutex=..., interval=...) at common/Cond.h:81 #3 0x00007fbb05dc3f7d in FileStore::sync_entry (this=0x7fbb10f90000) at os/filestore/FileStore.cc:3690 #4 0x00007fbb05def3ed in FileStore::SyncThread::entry (this=) at os/filestore/FileStore.h:185 #5 0x00007fbb03e1cdc5 in start_thread () from /lib64/libpthread.so.0 #6 0x00007fbb024a828d in ?? () from /lib64/libc.so.6 #7 0x0000000000000000 in ?? () Thread 13 (Thread 0x7fbaf0c9e700 (LWP 17000)): #0 0x00007fbb03e22f4d in __lll_lock_wait () from /lib64/libpthread.so.0 #1 0x00007fbb03e1ed02 in _L_lock_791 () from /lib64/libpthread.so.0 #2 0x00007fbb03e1ec08 in pthread_mutex_lock () from /lib64/libpthread.so.0 #3 0x00007fbb060b47d8 in Mutex::Lock (this=this@entry=0x7fbb1345d340, no_lockdep=no_lockdep@entry=false) at common/Mutex.cc:110 #4 0x00007fbb05a55665 in Locker (m=..., this=) at common/Mutex.h:115 #5 session_handle_reset (session=0x7fbb1345d260, this=0x7fbb1114a000) at osd/OSD.h:1438 #6 OSD::ms_handle_reset (this=0x7fbb1114a000, con=0x7fbb13187680) at osd/OSD.cc:4687 #7 0x00007fbb061c560f in ms_deliver_handle_reset (con=0x7fbb13187680, this=0x7fbb10fa4000) at msg/Messenger.h:657 #8 DispatchQueue::entry (this=0x7fbb10fa41d0) at msg/simple/DispatchQueue.cc:173 #9 0x00007fbb060def3d in DispatchQueue::DispatchThread::entry (this=) at msg/simple/DispatchQueue.h:103 #10 0x00007fbb03e1cdc5 in start_thread () from /lib64/libpthread.so.0 #11 0x00007fbb024a828d in ?? () from /lib64/libc.so.6 #12 0x0000000000000000 in ?? () Thread 12 (Thread 0x7fbafb649700 (LWP 16949)): #0 0x00007fbb03e206d5 in pthread_cond_wait@@GLIBC_2.3.2 () from /lib64/libpthread.so.0 #1 0x00007fbb060da13f in Wait (mutex=..., this=0x7fbb10fa4d60) at common/Cond.h:56 #2 SimpleMessenger::reaper_entry (this=0x7fbb10fa4700) at msg/simple/SimpleMessenger.cc:217 #3 0x00007fbb060dfa7d in SimpleMessenger::ReaperThread::entry (this=) at msg/simple/SimpleMessenger.h:196 #4 0x00007fbb03e1cdc5 in start_thread () from /lib64/libpthread.so.0 #5 0x00007fbb024a828d in ?? () from /lib64/libc.so.6 #6 0x0000000000000000 in ?? () Thread 11 (Thread 0x7fbaefc9c700 (LWP 17002)): #0 0x00007fbb0249dc3d in __statvfs_getflags () from /lib64/libc.so.6 #1 0x0000000000000000 in ?? () Thread 10 (Thread 0x7fbafc64b700 (LWP 16947)): #0 0x00007fbb03e22f4d in __lll_lock_wait () from /lib64/libpthread.so.0 #1 0x00007fbb03e1ed02 in _L_lock_791 () from /lib64/libpthread.so.0 #2 0x00007fbb03e1ec08 in pthread_mutex_lock () from /lib64/libpthread.so.0 #3 0x00007fbb060b47d8 in Mutex::Lock (this=this@entry=0x7fbb1114ae30, no_lockdep=no_lockdep@entry=false) at common/Mutex.cc:110 #4 0x00007fbb05a56b60 in OSD::heartbeat_reset (this=0x7fbb1114a000, con=0x7fbb13a60000) at osd/OSD.cc:4147 #5 0x00007fbb0622aac5 in ms_deliver_handle_reset (con=0x7fbb13a60000, this=0x7fbb10fac000) at msg/Messenger.h:657 #6 C_handle_reset::do_request (this=, id=) at msg/async/AsyncConnection.cc:87 #7 0x00007fbb061ca09a in EventCenter::process_events (this=this@entry=0x7fbb10fa8a48, timeout_microseconds=, timeout_microseconds@entry=30000000) at msg/async/Event.cc:406 ---Type to continue, or q to quit--- #8 0x00007fbb061a90e0 in Worker::entry (this=0x7fbb10fa8a00) at msg/async/AsyncMessenger.cc:294 #9 0x00007fbb03e1cdc5 in start_thread () from /lib64/libpthread.so.0 #10 0x00007fbb024a828d in ?? () from /lib64/libc.so.6 #11 0x0000000000000000 in ?? () Thread 9 (Thread 0x7fbaf9e46700 (LWP 16952)): #0 0x00007fbb03e206d5 in pthread_cond_wait@@GLIBC_2.3.2 () from /lib64/libpthread.so.0 #1 0x00007fbb060ec6ff in Wait (mutex=..., this=0x7fbb1114c588) at common/Cond.h:56 #2 SafeTimer::timer_thread (this=0x7fbb1114c578) at common/Timer.cc:116 #3 0x00007fbb060edd3d in SafeTimerThread::entry (this=) at common/Timer.cc:38 #4 0x00007fbb03e1cdc5 in start_thread () from /lib64/libpthread.so.0 #5 0x00007fbb024a828d in ?? () from /lib64/libc.so.6 #6 0x0000000000000000 in ?? () Thread 8 (Thread 0x7fbafe64f700 (LWP 16943)): #0 0x00007fbb03e22f4d in __lll_lock_wait () from /lib64/libpthread.so.0 #1 0x00007fbb03e1ed02 in _L_lock_791 () from /lib64/libpthread.so.0 #2 0x00007fbb03e1ec08 in pthread_mutex_lock () from /lib64/libpthread.so.0 #3 0x00007fbb060b47d8 in Mutex::Lock (this=this@entry=0x7ffd68175960, no_lockdep=no_lockdep@entry=false) at common/Mutex.cc:110 #4 0x00007fbb05cb7afa in Locker (m=..., this=) at common/Mutex.h:115 #5 sub_want (flags=1, start=35, what="", this=0x7ffd681757d0) at mon/MonClient.h:264 #6 Objecter::_maybe_request_map (this=this@entry=0x7fbb11158000) at osdc/Objecter.cc:1860 #7 0x00007fbb05cbf386 in Objecter::tick (this=0x7fbb11158000) at osdc/Objecter.cc:2056 #8 0x00007fbb05ae26d1 in operator() (this=0x7fbb10ee6850) at /usr/include/c++/4.8.2/functional:2471 #9 ceph::timer_detail::timer::timer_thread (this=0x7fbb111581f8) at common/ceph_timer.h:137 #10 0x00007fbb02d40220 in ?? () from /lib64/libstdc++.so.6 #11 0x00007fbb03e1cdc5 in start_thread () from /lib64/libpthread.so.0 #12 0x00007fbb024a828d in ?? () from /lib64/libc.so.6 #13 0x0000000000000000 in ?? () Thread 7 (Thread 0x7fbaff9bc700 (LWP 16940)): #0 0x00007fbb0249dc3d in __statvfs_getflags () from /lib64/libc.so.6 #1 0x0000000000000000 in ?? () Thread 6 (Thread 0x7fbafce4c700 (LWP 16946)): #0 0x00007fbb03e22f4d in __lll_lock_wait () from /lib64/libpthread.so.0 #1 0x00007fbb03e1ed02 in _L_lock_791 () from /lib64/libpthread.so.0 #2 0x00007fbb03e1ec08 in pthread_mutex_lock () from /lib64/libpthread.so.0 #3 0x00007fbb060b47d8 in Mutex::Lock (this=this@entry=0x7fbb1114ae30, no_lockdep=no_lockdep@entry=false) at common/Mutex.cc:110 #4 0x00007fbb05a76f6e in OSD::handle_osd_ping (this=this@entry=0x7fbb1114a000, m=m@entry=0x7fbb111ffc00) at osd/OSD.cc:3878 #5 0x00007fbb05a7866b in OSD::heartbeat_dispatch (this=0x7fbb1114a000, m=0x7fbb111ffc00) at osd/OSD.cc:5713 #6 0x00007fbb06213d9b in ms_fast_dispatch (m=, this=0x7fbb10fac000) at msg/Messenger.h:555 #7 AsyncConnection::DelayedDelivery::do_request (this=, id=) at msg/async/AsyncConnection.cc:2453 #8 0x00007fbb061c8626 in EventCenter::process_time_events (this=this@entry=0x7fbb10fa87c8) at msg/async/Event.cc:307 #9 0x00007fbb061c9d78 in EventCenter::process_events (this=this@entry=0x7fbb10fa87c8, timeout_microseconds=, timeout_microseconds@entry=30000000) at msg/async/Event.cc:392 #10 0x00007fbb061a90e0 in Worker::entry (this=0x7fbb10fa8780) at msg/async/AsyncMessenger.cc:294 #11 0x00007fbb03e1cdc5 in start_thread () from /lib64/libpthread.so.0 #12 0x00007fbb024a828d in ?? () from /lib64/libc.so.6 #13 0x0000000000000000 in ?? () Thread 5 (Thread 0x7fbafae48700 (LWP 16950)): #0 0x00007fbb03e22f4d in __lll_lock_wait () from /lib64/libpthread.so.0 #1 0x00007fbb03e254bc in _L_cond_lock_792 () from /lib64/libpthread.so.0 #2 0x00007fbb03e25398 in __pthread_mutex_cond_lock () from /lib64/libpthread.so.0 #3 0x00007fbb03e20b54 in pthread_cond_timedwait@@GLIBC_2.3.2 () from /lib64/libpthread.so.0 #4 0x00007fbb060ec99e in WaitUntil (when=..., mutex=..., this=0x7fbb1114a080) at common/Cond.h:72 #5 SafeTimer::timer_thread (this=0x7fbb1114a070) at common/Timer.cc:118 #6 0x00007fbb060edd3d in SafeTimerThread::entry (this=) at common/Timer.cc:38 #7 0x00007fbb03e1cdc5 in start_thread () from /lib64/libpthread.so.0 #8 0x00007fbb024a828d in ?? () from /lib64/libc.so.6 #9 0x0000000000000000 in ?? () Thread 4 (Thread 0x7fbafbe4a700 (LWP 16948)): #0 0x00007fbb03e206d5 in pthread_cond_wait@@GLIBC_2.3.2 () from /lib64/libpthread.so.0 #1 0x00007fbb060da13f in Wait (mutex=..., this=0x7fbb10fa5460) at common/Cond.h:56 #2 SimpleMessenger::reaper_entry (this=0x7fbb10fa4e00) at msg/simple/SimpleMessenger.cc:217 #3 0x00007fbb060dfa7d in SimpleMessenger::ReaperThread::entry (this=) at msg/simple/SimpleMessenger.h:196 #4 0x00007fbb03e1cdc5 in start_thread () from /lib64/libpthread.so.0 #5 0x00007fbb024a828d in ?? () from /lib64/libc.so.6 #6 0x0000000000000000 in ?? () Thread 3 (Thread 0x7fbafde4e700 (LWP 16944)): #0 0x00007fbb03e206d5 in pthread_cond_wait@@GLIBC_2.3.2 () from /lib64/libpthread.so.0 #1 0x00007fbb060da13f in Wait (mutex=..., this=0x7fbb10fa4660) at common/Cond.h:56 #2 SimpleMessenger::reaper_entry (this=0x7fbb10fa4000) at msg/simple/SimpleMessenger.cc:217 #3 0x00007fbb060dfa7d in SimpleMessenger::ReaperThread::entry (this=) at msg/simple/SimpleMessenger.h:196 #4 0x00007fbb03e1cdc5 in start_thread () from /lib64/libpthread.so.0 #5 0x00007fbb024a828d in ?? () from /lib64/libc.so.6 #6 0x0000000000000000 in ?? () Thread 2 (Thread 0x7fbafd64d700 (LWP 16945)): #0 0x00007fbb03e22f4d in __lll_lock_wait () from /lib64/libpthread.so.0 #1 0x00007fbb03e1ed02 in _L_lock_791 () from /lib64/libpthread.so.0 #2 0x00007fbb03e1ec08 in pthread_mutex_lock () from /lib64/libpthread.so.0 #3 0x00007fbb060b47d8 in Mutex::Lock (this=this@entry=0x7fbb1114ae30, no_lockdep=no_lockdep@entry=false) at common/Mutex.cc:110 #4 0x00007fbb05a76f6e in OSD::handle_osd_ping (this=this@entry=0x7fbb1114a000, m=m@entry=0x7fbb113b7600) at osd/OSD.cc:3878 ---Type to continue, or q to quit--- #5 0x00007fbb05a7866b in OSD::heartbeat_dispatch (this=0x7fbb1114a000, m=0x7fbb113b7600) at osd/OSD.cc:5713 #6 0x00007fbb06213d9b in ms_fast_dispatch (m=, this=0x7fbb10fac000) at msg/Messenger.h:555 #7 AsyncConnection::DelayedDelivery::do_request (this=, id=) at msg/async/AsyncConnection.cc:2453 #8 0x00007fbb061c8626 in EventCenter::process_time_events (this=this@entry=0x7fbb10fa8548) at msg/async/Event.cc:307 #9 0x00007fbb061c9d78 in EventCenter::process_events (this=this@entry=0x7fbb10fa8548, timeout_microseconds=, timeout_microseconds@entry=30000000) at msg/async/Event.cc:392 #10 0x00007fbb061a90e0 in Worker::entry (this=0x7fbb10fa8500) at msg/async/AsyncMessenger.cc:294 #11 0x00007fbb03e1cdc5 in start_thread () from /lib64/libpthread.so.0 #12 0x00007fbb024a828d in ?? () from /lib64/libc.so.6 #13 0x0000000000000000 in ?? () Thread 1 (Thread 0x7fbb001bd700 (LWP 16939)): #0 0x00007fbb03e23fcb in raise () from /lib64/libpthread.so.0 #1 0x00007fbb06006975 in reraise_fatal (signum=6) at global/signal_handler.cc:71 #2 handle_fatal_signal (signum=6) at global/signal_handler.cc:133 #3 #4 0x00007fbb023e75f7 in sigtimedwait () from /lib64/libc.so.6 #5 0x00007fbb001b95a8 in ?? () #6 0x0000000000000000 in ?? () (gdb) (gdb)