Actions
Bug #24008
closed[rbd-mirror] potential races during PoolReplayer shut-down
Status:
Resolved
Priority:
Normal
Assignee:
Jason Dillaman
Target version:
-
% Done:
0%
Source:
Tags:
Backport:
luminous
Regression:
No
Severity:
3 - minor
Reviewed:
Affected Versions:
ceph-qa-suite:
Pull request ID:
Crash signature (v1):
Crash signature (v2):
Description
#0 pthread_cond_wait@@GLIBC_2.3.2 () at ../sysdeps/unix/sysv/linux/x86_64/pthread_cond_wait.S:185 #1 0x0000555b809347b0 in Cond::Wait (mutex=..., this=0x7ffcb014a878) at /build/ceph-13.0.2-2060-gde9606d/src/common/Cond.h:48 #2 C_SaferCond::wait (this=0x7ffcb014a800) at /build/ceph-13.0.2-2060-gde9606d/src/common/Cond.h:195 #3 rbd::mirror::InstanceReplayer<librbd::ImageCtx>::shut_down (this=<optimized out>) at /build/ceph-13.0.2-2060-gde9606d/src/tools/rbd_mirror/InstanceReplayer.cc:79 #4 0x0000555b80908143 in rbd::mirror::PoolReplayer<librbd::ImageCtx>::shut_down (this=0x555b82921890) at /build/ceph-13.0.2-2060-gde9606d/src/tools/rbd_mirror/PoolReplayer.cc:387 #5 0x0000555b8090f065 in rbd::mirror::PoolReplayer<librbd::ImageCtx>::~PoolReplayer (this=0x555b82921890, __in_chrg=<optimized out>) at /build/ceph-13.0.2-2060-gde9606d/src/tools/rbd_mirror/PoolReplayer.cc:250 #6 0x0000555b8090324d in std::default_delete<rbd::mirror::PoolReplayer<librbd::ImageCtx> >::operator() (this=<optimized out>, __ptr=0x555b82921890) at /usr/include/c++/7/bits/unique_ptr.h:78 #7 std::unique_ptr<rbd::mirror::PoolReplayer<librbd::ImageCtx>, std::default_delete<rbd::mirror::PoolReplayer<librbd::ImageCtx> > >::~unique_ptr (this=0x555b82b34828, __in_chrg=<optimized out>) at /usr/include/c++/7/bits/unique_ptr.h:268 #8 std::pair<std::pair<long, rbd::mirror::peer_t> const, std::unique_ptr<rbd::mirror::PoolReplayer<librbd::ImageCtx>, std::default_delete<rbd::mirror::PoolReplayer<librbd::ImageCtx> > > >::~pair ( this=0x555b82b347c0, __in_chrg=<optimized out>) at /usr/include/c++/7/bits/stl_pair.h:198 #9 __gnu_cxx::new_allocator<std::_Rb_tree_node<std::pair<std::pair<long, rbd::mirror::peer_t> const, std::unique_ptr<rbd::mirror::PoolReplayer<librbd::ImageCtx>, std::default_delete<rbd::mirror::PoolReplayer<librbd::ImageCtx> > > > > >::destroy<std::pair<std::pair<long, rbd::mirror::peer_t> const, std::unique_ptr<rbd::mirror::PoolReplayer<librbd::ImageCtx>, std::default_delete<rbd::mirror::PoolReplayer<librbd::ImageCtx> > > > > (this=<optimized out>, __p=<optimized out>) at /usr/include/c++/7/ext/new_allocator.h:140 #10 std::allocator_traits<std::allocator<std::_Rb_tree_node<std::pair<std::pair<long, rbd::mirror::peer_t> const, std::unique_ptr<rbd::mirror::PoolReplayer<librbd::ImageCtx>, std::default_delete<rbd::mirror::PoolReplayer<librbd::ImageCtx> > > > > > >::destroy<std::pair<std::pair<long, rbd::mirror::peer_t> const, std::unique_ptr<rbd::mirror::PoolReplayer<librbd::ImageCtx>, std::default_delete<rbd::mirror::PoolReplayer<librbd::ImageCtx> > > > > (__a=..., __p=<optimized out>) at /usr/include/c++/7/bits/alloc_traits.h:487 #11 std::_Rb_tree<std::pair<long, rbd::mirror::peer_t>, std::pair<std::pair<long, rbd::mirror::peer_t> const, std::unique_ptr<rbd::mirror::PoolReplayer<librbd::ImageCtx>, std::default_delete<rbd::mirror::PoolReplayer<librbd::ImageCtx> > > >, std::_Select1st<std::pair<std::pair<long, rbd::mirror::peer_t> const, std::unique_ptr<rbd::mirror::PoolReplayer<librbd::ImageCtx>, std::default_delete<rbd::mirror::PoolReplayer<librbd::ImageCtx> > > > >, std::less<std::pair<long, rbd::mirror::peer_t> >, std::allocator<std::pair<std::pair<long, rbd::mirror::peer_t> const, std::unique_ptr<rbd::mirror::PoolReplayer<librbd::ImageCtx>, std::default_delete<rbd::mirror::PoolReplayer<librbd::ImageCtx> > > > > >::_M_destroy_node (this=0x555b82854898, __p=0x555b82b347a0) at /usr/include/c++/7/bits/stl_tree.h:650 #12 std::_Rb_tree<std::pair<long, rbd::mirror::peer_t>, std::pair<std::pair<long, rbd::mirror::peer_t> const, std::unique_ptr<rbd::mirror::PoolReplayer<librbd::ImageCtx>, std::default_delete<rbd::mirror::PoolReplayer<librbd::ImageCtx> > > >, std::_Select1st<std::pair<std::pair<long, rbd::mirror::peer_t> const, std::unique_ptr<rbd::mirror::PoolReplayer<librbd::ImageCtx>, std::default_delete<rbd::mirror::PoolReplayer<librbd::ImageCtx> > > > >, std::less<std::pair<long, rbd::mirror::peer_t> >, std::allocator<std::pair<std::pair<long, rbd::mirror::peer_t> const, std::unique_ptr<rbd::mirror::PoolReplayer<librbd::ImageCtx>, std::default_delete<rbd::mirror::PoolReplayer<librbd::ImageCtx> > > > > >::_M_drop_node (this=0x555b82854898, __p=0x555b82b347a0) at /usr/include/c++/7/bits/stl_tree.h:658 #13 std::_Rb_tree<std::pair<long, rbd::mirror::peer_t>, std::pair<std::pair<long, rbd::mirror::peer_t> const, std::unique_ptr<rbd::mirror::PoolReplayer<librbd::ImageCtx>, std::default_delete<rbd::mirror::PoolReplayer<librbd::ImageCtx> > > >, std::_Select1st<std::pair<std::pair<long, rbd::mirror::peer_t> const, std::unique_ptr<rbd::mirror::PoolReplayer<librbd::ImageCtx>, std::default_delete<rbd::mirror::PoolReplayer<librbd::ImageCtx> > > > >, std::less<std::pair<long, rbd::mirror::peer_t> >, std::allocator<std::pair<std::pair<long, rbd::mirror::peer_t> const, std::unique_ptr<rbd::mirror::PoolReplayer<librbd::ImageCtx>, std::default_delete<rbd::mirror::PoolReplayer<librbd::ImageCtx> > > > > >::_M_erase (this=this@entry=0x555b82854898, __x=0x555b82b347a0) at /usr/include/c++/7/bits/stl_tree.h:1858 #14 0x0000555b80901530 in std::_Rb_tree<std::pair<long, rbd::mirror::peer_t>, std::pair<std::pair<long, rbd::mirror::peer_t> const, std::unique_ptr<rbd::mirror::PoolReplayer<librbd::ImageCtx>, std::default_delete<rbd::mirror::PoolReplayer<librbd::ImageCtx> > > >, std::_Select1st<std::pair<std::pair<long, rbd::mirror::peer_t> const, std::unique_ptr<rbd::mirror::PoolReplayer<librbd::ImageCtx>, std::default_delete<rbd::mirror::PoolReplayer<librbd::ImageCtx> > > > >, std::less<std::pair<long, rbd::mirror::peer_t> >, std::allocator<std::pair<std::pair<long, rbd::mirror::peer_t> const, std::unique_ptr<rbd::mirror::PoolReplayer<librbd::ImageCtx>, std::default_delete<rbd::mirror::PoolReplayer<librbd::ImageCtx> > > > > >::~_Rb_tree (this=0x555b82854898, __in_chrg=<optimized out>) at /usr/include/c++/7/bits/stl_tree.h:949 #15 std::map<std::pair<long, rbd::mirror::peer_t>, std::unique_ptr<rbd::mirror::PoolReplayer<librbd::ImageCtx>, std::default_delete<rbd::mirror::PoolReplayer<librbd::ImageCtx> > >, std::less<std::pair<long, rbd::mirror::peer_t> >, std::allocator<std::pair<std::pair<long, rbd::mirror::peer_t> const, std::unique_ptr<rbd::mirror::PoolReplayer<librbd::ImageCtx>, std::default_delete<rbd::mirror::PoolReplayer<librbd::ImageCtx> > > > > >::~map (this=0x555b82854898, __in_chrg=<optimized out>) at /usr/include/c++/7/bits/stl_map.h:294 #16 rbd::mirror::Mirror::~Mirror (this=0x555b828547a0, __in_chrg=<optimized out>) at /build/ceph-13.0.2-2060-gde9606d/src/tools/rbd_mirror/Mirror.cc:213 #17 0x0000555b808f0770 in main (argc=<optimized out>, argv=<optimized out>) at /build/ceph-13.0.2-2060-gde9606d/src/tools/rbd_mirror/main.cc:82 #0 0x00007f7d386d0269 in raise (sig=sig@entry=6) at ../sysdeps/unix/sysv/linux/pt-raise.c:35 #1 0x0000555b80bf8abe in reraise_fatal (signum=6) at /build/ceph-13.0.2-2060-gde9606d/src/global/signal_handler.cc:74 #2 handle_fatal_signal (signum=6) at /build/ceph-13.0.2-2060-gde9606d/src/global/signal_handler.cc:138 #3 <signal handler called> #4 0x00007f7d37f1e428 in __GI_raise (sig=sig@entry=6) at ../sysdeps/unix/sysv/linux/raise.c:54 #5 0x00007f7d37f2002a in __GI_abort () at abort.c:89 #6 0x00007f7d38bc0250 in ceph::__ceph_assert_fail (assertion=<optimized out>, file=<optimized out>, line=<optimized out>, func=<optimized out>) at /build/ceph-13.0.2-2060-gde9606d/src/common/assert.cc:66 #7 0x00007f7d38bc02c7 in ceph::__ceph_assert_fail (ctx=...) at /build/ceph-13.0.2-2060-gde9606d/src/common/assert.cc:71 #8 0x00007f7d38b9502e in Mutex::Lock (this=this@entry=0x555b82b2f6f0, no_lockdep=no_lockdep@entry=false) at /build/ceph-13.0.2-2060-gde9606d/src/common/Mutex.cc:110 #9 0x0000555b80947ce0 in Mutex::Locker::Locker (m=..., this=<synthetic pointer>) at /build/ceph-13.0.2-2060-gde9606d/src/common/Mutex.h:115 #10 rbd::mirror::InstanceWatcher<librbd::ImageCtx>::notify_sync_request (this=0x555b82b2f4a0, sync_id="1074661a00d5", on_sync_start=0x7f7c6c001bb0) at /build/ceph-13.0.2-2060-gde9606d/src/tools/rbd_mirror/InstanceWatcher.cc:473 #11 0x0000555b809a4f95 in rbd::mirror::ImageSync<librbd::ImageCtx>::send_notify_sync_request (this=0x7f7c6c036a50) at /build/ceph-13.0.2-2060-gde9606d/src/tools/rbd_mirror/ImageSync.cc:103 #12 0x0000555b8098aa06 in rbd::mirror::image_replayer::BootstrapRequest<librbd::ImageCtx>::image_sync (this=0x7f7d2007c560) at /build/ceph-13.0.2-2060-gde9606d/src/tools/rbd_mirror/image_replayer/BootstrapRequest.cc:667 #13 0x0000555b8098ae15 in rbd::mirror::image_replayer::BootstrapRequest<librbd::ImageCtx>::get_remote_tags (this=this@entry=0x7f7d2007c560) at /build/ceph-13.0.2-2060-gde9606d/src/tools/rbd_mirror/image_replayer/BootstrapRequest.cc:487 #14 0x0000555b8098d208 in rbd::mirror::image_replayer::BootstrapRequest<librbd::ImageCtx>::handle_open_local_image (this=0x7f7d2007c560, r=0) at /build/ceph-13.0.2-2060-gde9606d/src/tools/rbd_mirror/image_replayer/BootstrapRequest.cc:333 #15 0x0000555b80905319 in Context::complete (this=0x7f7cd00238a0, r=<optimized out>) at /build/ceph-13.0.2-2060-gde9606d/src/include/Context.h:77 #16 0x0000555b8099b701 in rbd::mirror::image_replayer::OpenLocalImageRequest<librbd::ImageCtx>::finish (this=0x7f7cd00977b0, r=0) at /build/ceph-13.0.2-2060-gde9606d/src/tools/rbd_mirror/image_replayer/OpenLocalImageRequest.cc:251 #17 0x0000555b8099c030 in rbd::mirror::image_replayer::OpenLocalImageRequest<librbd::ImageCtx>::handle_lock_image (this=0x7f7cd00977b0, r=<optimized out>) at /build/ceph-13.0.2-2060-gde9606d/src/tools/rbd_mirror/image_replayer/OpenLocalImageRequest.cc:220 #18 0x0000555b80905319 in Context::complete (this=0x7f7c6c00abc0, r=<optimized out>) at /build/ceph-13.0.2-2060-gde9606d/src/include/Context.h:77 #19 0x0000555b80a3657d in librbd::ManagedLock<librbd::ImageCtx>::complete_active_action (this=this@entry=0x7f7cd001aaa0, next_state=librbd::ManagedLock<librbd::ImageCtx>::STATE_LOCKED, r=r@entry=0) at /build/ceph-13.0.2-2060-gde9606d/src/librbd/ManagedLock.cc:433 #20 0x0000555b80a370e6 in librbd::ManagedLock<librbd::ImageCtx>::handle_post_acquire_lock (this=0x7f7cd001aaa0, r=0) at /build/ceph-13.0.2-2060-gde9606d/src/librbd/ManagedLock.cc:533 #21 0x0000555b80905319 in Context::complete (this=0x7f7c6c007ff0, r=<optimized out>) at /build/ceph-13.0.2-2060-gde9606d/src/include/Context.h:77 #22 0x0000555b809e6190 in librbd::ExclusiveLock<librbd::ImageCtx>::handle_post_acquired_lock (this=0x7f7cd001aaa0, r=0) at /build/ceph-13.0.2-2060-gde9606d/src/librbd/ExclusiveLock.cc:284 #23 0x0000555b80905319 in Context::complete (this=0x7f7c6c01c7e0, r=<optimized out>) at /build/ceph-13.0.2-2060-gde9606d/src/include/Context.h:77 #24 0x00007f7d38bc61b7 in ThreadPool::worker (this=0x7f7cd0012d80, wt=<optimized out>) at /build/ceph-13.0.2-2060-gde9606d/src/common/WorkQueue.cc:120 #25 0x00007f7d38bc70e0 in ThreadPool::WorkThread::entry (this=<optimized out>) at /build/ceph-13.0.2-2060-gde9606d/src/common/WorkQueue.h:448 #26 0x00007f7d386c66ba in start_thread (arg=0x7f7c857fa700) at pthread_create.c:333 #27 0x00007f7d37ff041d in clone () at ../sysdeps/unix/sysv/linux/x86_64/clone.S:109
Actions