Ceph » rbd

This was encountered on an infernalis build:

#0  0x00007f037744489d in __lll_lock_wait () from /lib64/libpthread.so.0
#1  0x00007f037743e9cd in pthread_mutex_lock () from /lib64/libpthread.so.0
#2  0x00007f037a8f5c16 in Mutex::Lock (this=0x55c9c7a8d2f0, no_lockdep=<optimized out>) at common/Mutex.cc:102
#3  0x00007f037a85d683 in Mutex::Locker::Locker (m=..., this=<synthetic pointer>) at ./common/Mutex.h:115
#4  librbd::C_OrderedWrite::finish (this=0x7f0344307330, r=0) at librbd/LibrbdWriteback.cc:80
#5  0x00007f037a7f4a91 in Context::complete(int) () from /home/jdillaman/ceph_infernalis/src/.libs/librbd.so.1
#6  0x00007f037a7fc667 in librbd::AioRequest::complete (this=0x7f0344157100, r=0) at librbd/AioRequest.cc:51

#7  0x00007f03782bda18 in librados::C_AioSafe::finish (this=0x7f034c01f410, r=<optimized out>) at ./librados/AioCompletionImpl.h:199
#8  0x00007f037829d349 in Context::complete (this=0x7f034c01f410, r=<optimized out>) at ./include/Context.h:64
#9  0x00007f0378361b30 in Finisher::finisher_thread_entry (this=0x55c9c7a7f790) at common/Finisher.cc:65
#10 0x00007f037743c60a in start_thread () from /lib64/libpthread.so.0
#11 0x00007f03766d3a4d in clone () from /lib64/libc.so.6

Thread 3 (Thread 0x7f03527fc700 (LWP 28217)):
#0  0x00007f0377441b10 in pthread_cond_wait@@GLIBC_2.3.2 () from /lib64/libpthread.so.0
#1  0x00007f037835a8c3 in Cond::Wait (mutex=..., this=0x7f03481e1770) at ./common/Cond.h:55
#2  Throttle::_wait (this=this@entry=0x55c9c7a81448, c=c@entry=1) at common/Throttle.cc:106
#3  0x00007f037835b6f5 in Throttle::get (this=0x55c9c7a81448, c=1, m=0) at common/Throttle.cc:176
#4  0x00007f03782cc634 in Objecter::_throttle_op (this=<optimized out>, op=<optimized out>, op_budget=4096) at osdc/Objecter.cc:2977
#5  0x00007f03782e0b7d in Objecter::_take_op_budget (op=0x7f0348589e10, this=0x55c9c7a80e30) at osdc/Objecter.h:1872
#6  Objecter::_op_submit_with_budget (this=this@entry=0x55c9c7a80e30, op=op@entry=0x7f0348589e10, lc=..., ctx_budget=ctx_budget@entry=0x0)
    at osdc/Objecter.cc:2053
#7  0x00007f03782e0c9f in Objecter::op_submit (this=this@entry=0x55c9c7a80e30, op=op@entry=0x7f0348589e10, ctx_budget=ctx_budget@entry=0x0)
    at osdc/Objecter.cc:2039
#8  0x00007f03782b3fac in Objecter::mutate (reqid=..., objver=0x7f0348589d40, oncommit=0x7f0348288b80, onack=0x7f034828a5c0, flags=0, 
    mtime=..., snapc=..., op=..., oloc=..., oid=..., this=0x55c9c7a80e30) at ./osdc/Objecter.h:2104
#9  librados::IoCtxImpl::aio_operate (this=0x55c9c7a8dab0, oid=..., o=0x7f0348429040, c=0x7f0348589c80, snap_context=..., 
    flags=flags@entry=0) at librados/IoCtxImpl.cc:603
#10 0x00007f0378294af1 in librados::IoCtx::aio_operate (this=0x55c9c7a8d1d0, oid="rbd_data.100612949cc4.", '0' <repeats 14 times>, "3b", 
    c=c@entry=0x7f034828bc20, o=o@entry=0x7f03483d2958, snap_seq=4, snaps=std::vector of length 1, capacity 1 = {...})
    at librados/librados.cc:1385
#11 0x00007f037a7fd918 in librbd::AbstractWrite::send_write (this=this@entry=0x7f03483d28e0) at librbd/AioRequest.cc:477
#12 0x00007f037a7fdd62 in librbd::AbstractWrite::send_pre (this=this@entry=0x7f03483d28e0) at librbd/AioRequest.cc:431
#13 0x00007f037a7fe11e in librbd::AbstractWrite::send (this=this@entry=0x7f03483d28e0) at librbd/AioRequest.cc:392
#14 0x00007f037a85c107 in librbd::LibrbdWriteback::write (this=0x55c9c7a9f6f0, oid=..., oloc=..., off=off@entry=2367488, 
    len=len@entry=4096, snapc=..., bl=..., mtime=..., trunc_size=0, trunc_seq=0, oncommit=0x7f0348428de0) at librbd/LibrbdWriteback.cc:178
#15 0x00007f037ab7dae9 in ObjectCacher::bh_write (this=this@entry=0x55c9c7a9ff80, bh=bh@entry=0x7f03550db2e0) at osdc/ObjectCacher.cc:849
#16 0x00007f037ab8a7ef in ObjectCacher::flush_set (this=0x55c9c7a9ff80, oset=0x55c9c7a88760, onfinish=0x7f03527fb610)
    at osdc/ObjectCacher.cc:1745
#17 0x00007f037a8174d1 in librbd::ImageCtx::flush_cache (this=0x55c9c7a8d010, onfinish=0x7f03527fb610) at librbd/ImageCtx.cc:752
#18 0x00007f037a8175a8 in librbd::ImageCtx::flush_cache (this=this@entry=0x55c9c7a8d010) at librbd/ImageCtx.cc:740
---Type <return> to continue, or q <return> to quit---
#19 0x00007f037a818905 in librbd::ImageCtx::flush (this=0x55c9c7a8d010) at librbd/ImageCtx.cc:853
#20 0x00007f037a82a625 in librbd::ImageWatcher::release_lock (this=this@entry=0x55c9c7aa0430) at librbd/ImageWatcher.cc:392
#21 0x00007f037a82c2b0 in librbd::ImageWatcher::reregister_watch (this=0x55c9c7aa0430) at librbd/ImageWatcher.cc:1076
#22 0x00007f037a80144c in boost::function1<void, int>::operator() (a0=<optimized out>, this=<optimized out>)
    at /usr/include/boost/function/function_template.hpp:773
#23 FunctionContext::finish (this=<optimized out>, r=<optimized out>) at ./include/Context.h:460
#24 0x00007f037a7f4a91 in Context::complete(int) () from /home/jdillaman/ceph_infernalis/src/.libs/librbd.so.1
#25 0x00007f037a7f4a91 in Context::complete(int) () from /home/jdillaman/ceph_infernalis/src/.libs/librbd.so.1
#26 0x00007f037a892360 in Finisher::finisher_thread_entry (this=0x55c9c7aa06d0) at common/Finisher.cc:65
#27 0x00007f037743c60a in start_thread () from /lib64/libpthread.so.0
#28 0x00007f03766d3a4d in clone () from /lib64/libc.so.6