Project

General

Profile

Actions

Bug #56519

closed

osd crash due to non-controlled concurrent backref cache access

Added by Xuehan Xu almost 2 years ago. Updated over 1 year ago.

Status:
Resolved
Priority:
Normal
Assignee:
Target version:
-
% Done:

0%

Source:
Tags:
Backport:
Regression:
No
Severity:
3 - minor
Reviewed:
Affected Versions:
ceph-qa-suite:
Pull request ID:
Crash signature (v1):
Crash signature (v2):

Description

DEBUG 2022-07-05 18:18:30,814 [shard 0] seastore_tm - 0x1129c8a0 TransactionManager::alloc_extent: new extent: CachedExtent(addr=0x86e56c0, type=OBJECT_DATA_BLOCK, version=0, dirty_from_or_retired_at=JOURNAL_SEQ_NULL, modify_time=tp(NULL), paddr=paddr_t<DELAYED_TEMP>, length=4096, state=INITIAL_WRITE_PENDING, last_committed_crc=0, refcount=2, user_hint=HOT, reclaim_gen=^@, laddr=18375058212384129024, pin=LBAPin(18375058212384129024~4096->paddr_t<DELAYED_TEMP>), laddr_hint: 18375058212384129024
DEBUG 2022-07-05 18:18:30,814 [shard 0] seastore_backref - 0x11ae6b20 BtreeBackrefManager::new_mapping: 18375146822111105024~4096, paddr=paddr_t<Seg[0,9], 50872320>, state: paddr_t<Seg[0,10], 8192>~4096, insert at paddr_t<Seg[0,9], 50872320> -- backref_map_val_t(18375146053491019776~4096)

....
DEBUG 2022-07-05 18:18:31,009 [shard 0] seastore_cache - 0x11661010 Cache::retire_extent: retire extent -- CachedExtent(addr=0x11a67220, type=LADDR_LEAF, version=10, dirty_from_or_retired_at=journal_seq_t(segment_seq=13, offset=paddr_t<Seg[0,9], 57696256>), modify_time=tp(2022-07-05 18:18:09), paddr=paddr_t<Seg[0,9], 50872320>, length=4096, state=DIRTY, last_committed_crc=565107345, refcount=7, user_hint=REWRITE, reclaim_gen=^A, size=90, meta=btree_node_meta_t(begin=18375146822111105024, end=18375146894441447424, depth=1))

....
DEBUG 2022-07-05 18:18:31,024 [shard 0] seastore_cache - 0x11661010 Cache::prepare_record: retired and remove extent -- CachedExtent(addr=0x11a67220, type=LADDR_LEAF, version=10, dirty_from_or_retired_at=journal_seq_t(segment_seq=13, offset=paddr_t<Seg[0,9], 57696256>), modify_time=tp(2022-07-05 18:18:09), paddr=paddr_t<Seg[0,9], 50872320>, length=4096, state=DIRTY, last_committed_crc=565107345, refcount=4, user_hint=REWRITE, reclaim_gen=^A, size=90, meta=btree_node_meta_t(begin=18375146822111105024, end=18375146894441447424, depth=1))
DEBUG 2022-07-05 18:18:31,024 [shard 0] seastore_cache - 0x11661010 Cache::prepare_record: adding release alloc_blk: paddr_t<Seg[0,9], 50872320>~4096, type: LADDR_LEAF, seq:13

....
DEBUG 2022-07-05 18:18:31,070 [shard 0] seastore_cache - Cache::trim_backref_bufs: trimming to journal_seq_t(segment_seq=13, offset=paddr_t<Seg[0,9], 52015104>)
DEBUG 2022-07-05 18:18:31,070 [shard 0] seastore_cache - 0x11ae6b20 Cache::prepare_record: invalid mutated extent -- CachedExtent(addr=0x173801b0, type=BACKREF_LEAF, version=6, dirty_from_or_retired_at=journal_seq_t(segment_seq=15, offset=paddr_t<Seg[0,27], 50176000>), modify_time=tp(NULL), paddr=paddr_t<Seg[0,27], 50167808>, length=4096, state=INVALID, last_committed_crc=2979365815, refcount=1, user_hint=REWRITE, reclaim_gen=^A)
DEBUG 2022-07-05 18:18:31,070 [shard 0] seastore_cache - 0x11ae6b20 Cache::prepare_record: invalid mutated extent -- CachedExtent(addr=0xaba6320, type=BACKREF_LEAF, version=21, dirty_from_or_retired_at=journal_seq_t(segment_seq=15, offset=paddr_t<Seg[0,27], 48947200>), modify_time=tp(NULL), paddr=paddr_t<Seg[0,27], 48734208>, length=4096, state=INVALID, last_committed_crc=1331229339, refcount=1, user_hint=REWRITE, reclaim_gen=^A)

....
crimson-osd: ../src/crimson/os/seastore/cache.h:623: crimson::os::seastore::backref_buf_entry_t& crimson::os::seastore::Cache::get_del_backref(crimson::os::seastore::paddr_t): Assertion `it != backref_remove_set.end()' failed.
Aborting on shard 0.
Backtrace:
 0# gsignal in /lib64/libc.so.6
 1# abort in /lib64/libc.so.6
 2# 0x00002B0D521A39F9 in /lib64/libc.so.6
 3# 0x00002B0D521B1CC6 in /lib64/libc.so.6
 4# crimson::os::seastore::backref::BtreeBackrefManager::get_cached_backref_removal(crimson::os::seastore::paddr_t) at /home/xuxuehan/nvme/ceph/build/../src/crimson/os/seastore/backref/btree_backref_manager.cc:503
 5# operator()<boost::intrusive_ptr<crimson::os::seastore::CachedExtent> > at /home/xuxuehan/nvme/ceph/build/../src/crimson/os/seastore/async_cleaner.cc:927
 6# auto crimson::interruptible::internal::call_with_interruption_impl<crimson::os::seastore::TransactionConflictCondition, crimson::os::seastore::AsyncCleaner::_retrieve_live_extents(crimson::os::seastore::Transaction&, std::set<crimson::os::seastore::backref_buf_entry_t*, crimson::os::seastore::backref_buf_entry_t::cmp_t, std::allocator<crimson::os::seastore::backref_buf_entry_t*> >&&, std::vector<boost::intrusive_ptr<crimson::os::seastore::CachedExtent>, std::allocator<boost::intrusive_ptr<crimson::os::seastore::CachedExtent> > >&, std::map<crimson::os::seastore::journal_seq_t, std::__cxx11::list<crimson::os::seastore::backref_buf_entry_t*, std::allocator<crimson::os::seastore::backref_buf_entry_t*> >, std::less<crimson::os::seastore::journal_seq_t>, std::allocator<std::pair<crimson::os::seastore::journal_seq_t const, std::__cxx11::list<crimson::os::seastore::backref_buf_entry_t*, std::allocator<crimson::os::seastore::backref_buf_entry_t*> > > > >&)::{lambda(auto:1&)#1}::operator()<std::set<crimson::os::seastore::backref_buf_entry_t*, crimson::os::seastore::backref_buf_entry_t::cmp_t, std::allocator<crimson::os::seastore::backref_buf_entry_t*> > >(std::set<crimson::os::seastore::backref_buf_entry_t*, crimson::os::seastore::backref_buf_entry_t::cmp_t, std::allocator<crimson::os::seastore::backref_buf_entry_t*> >&) const::{lambda(auto:1&)#1}::operator()<crimson::os::seastore::backref_buf_entry_t* const>(std::set<crimson::os::seastore::backref_buf_entry_t*, crimson::os::seastore::backref_buf_entry_t::cmp_t, std::allocator<crimson::os::seastore::backref_buf_entry_t*> >&) const::{lambda(auto:1)#1}, boost::intrusive_ptr<crimson::os::seastore::CachedExtent> >(seastar::lw_shared_ptr<crimson::os::seastore::TransactionConflictCondition>, crimson::os::seastore::AsyncCleaner::_retrieve_live_extents(crimson::os::seastore::Transaction&, std::set<crimson::os::seastore::backref_buf_entry_t*, crimson::os::seastore::backref_buf_entry_t::cmp_t, std::allocator<crimson::os::seastore::backref_buf_entry_t*> >&&, std::vector<boost::intrusive_ptr<crimson::os::seastore::CachedExtent>, std::allocator<boost::intrusive_ptr<crimson::os::seastore::CachedExtent> > >&, std::map<crimson::os::seastore::journal_seq_t, std::__cxx11::list<crimson::os::seastore::backref_buf_entry_t*, std::allocator<crimson::os::seastore::backref_buf_entry_t*> >, std::less<crimson::os::seastore::journal_seq_t>, std::allocator<std::pair<crimson::os::seastore::journal_seq_t const, std::__cxx11::list<crimson::os::seastore::backref_buf_entry_t*, std::allocator<crimson::os::seastore::backref_buf_entry_t*> > > > >&)::{lambda(auto:1&)#1}::operator()<std::set<crimson::os::seastore::backref_buf_entry_t*, crimson::os::seastore::backref_buf_entry_t::cmp_t, std::allocator<crimson::os::seastore::backref_buf_entry_t*> > >(std::set<crimson::os::seastore::backref_buf_entry_t*, crimson::os::seastore::backref_buf_entry_t::cmp_t, std::allocator<crimson::os::seastore::backref_buf_entry_t*> >&) const::{lambda(auto:1&)#1}::operator()<crimson::os::seastore::backref_buf_entry_t* const>(std::set<crimson::os::seastore::backref_buf_entry_t*, crimson::os::seastore::backref_buf_entry_t::cmp_t, std::allocator<crimson::os::seastore::backref_buf_entry_t*> >&) const::{lambda(auto:1)#1}&&, boost::intrusive_ptr<crimson::os::seastore::CachedExtent>&&) at /home/xuxuehan/nvme/ceph/build/../src/crimson/common/interruptible_future.h:193
 7# auto crimson::interruptible::call_with_interruption<crimson::os::seastore::TransactionConflictCondition, crimson::os::seastore::AsyncCleaner::_retrieve_live_extents(crimson::os::seastore::Transaction&, std::set<crimson::os::seastore::backref_buf_entry_t*, crimson::os::seastore::backref_buf_entry_t::cmp_t, std::allocator<crimson::os::seastore::backref_buf_entry_t*> >&&, std::vector<boost::intrusive_ptr<crimson::os::seastore::CachedExtent>, std::allocator<boost::intrusive_ptr<crimson::os::seastore::CachedExtent> > >&, std::map<crimson::os::seastore::journal_seq_t, std::__cxx11::list<crimson::os::seastore::backref_buf_entry_t*, std::allocator<crimson::os::seastore::backref_buf_entry_t*> >, std::less<crimson::os::seastore::journal_seq_t>, std::allocator<std::pair<crimson::os::seastore::journal_seq_t const, std::__cxx11::list<crimson::os::seastore::backref_buf_entry_t*, std::allocator<crimson::os::seastore::backref_buf_entry_t*> > > > >&)::{lambda(auto:1&)#1}::operator()<std::set<crimson::os::seastore::backref_buf_entry_t*, crimson::os::seastore::backref_buf_entry_t::cmp_t, std::allocator<crimson::os::seastore::backref_buf_entry_t*> > >(std::set<crimson::os::seastore::backref_buf_entry_t*, crimson::os::seastore::backref_buf_entry_t::cmp_t, std::allocator<crimson::os::seastore::backref_buf_entry_t*> >&) const::{lambda(auto:1&)#1}::operator()<crimson::os::seastore::backref_buf_entry_t* const>(std::set<crimson::os::seastore::backref_buf_entry_t*, crimson::os::seastore::backref_buf_entry_t::cmp_t, std::allocator<crimson::os::seastore::backref_buf_entry_t*> >&) const::{lambda(auto:1)#1}, boost::intrusive_ptr<crimson::os::seastore::CachedExtent>, crimson::interruptible::interruptible_future_detail<crimson::os::seastore::TransactionConflictCondition, crimson::errorator<crimson::unthrowable_wrapper<std::error_code const&, crimson::ec<5> > >::_future<crimson::errorated_future_marker<void> > >, 0>(seastar::lw_shared_ptr<crimson::os::seastore::TransactionConflictCondition>, crimson::os::seastore::AsyncCleaner::_retrieve_live_extents(crimson::os::seastore::Transaction&, std::set<crimson::os::seastore::backref_buf_entry_t*, crimson::os::seastore::backref_buf_entry_t::cmp_t, std::allocator<crimson::os::seastore::backref_buf_entry_t*> >&&, std::vector<boost::intrusive_ptr<crimson::os::seastore::CachedExtent>, std::allocator<boost::intrusive_ptr<crimson::os::seastore::CachedExtent> > >&, std::map<crimson::os::seastore::journal_seq_t, std::__cxx11::list<crimson::os::seastore::backref_buf_entry_t*, std::allocator<crimson::os::seastore::backref_buf_entry_t*> >, std::less<crimson::os::seastore::journal_seq_t>, std::allocator<std::pair<crimson::os::seastore::journal_seq_t const, std::__cxx11::list<crimson::os::seastore::backref_buf_entry_t*, std::allocator<crimson::os::seastore::backref_buf_entry_t*> > > > >&)::{lambda(auto:1&)#1}::operator()<std::set<crimson::os::seastore::backref_buf_entry_t*, crimson::os::seastore::backref_buf_entry_t::cmp_t, std::allocator<crimson::os::seastore::backref_buf_entry_t*> > >(std::set<crimson::os::seastore::backref_buf_entry_t*, crimson::os::seastore::backref_buf_entry_t::cmp_t, std::allocator<crimson::os::seastore::backref_buf_entry_t*> >&) const::{lambda(auto:1&)#1}::operator()<crimson::os::seastore::backref_buf_entry_t* const>(std::set<crimson::os::seastore::backref_buf_entry_t*, crimson::os::seastore::backref_buf_entry_t::cmp_t, std::allocator<crimson::os::seastore::backref_buf_entry_t*> >&) const::{lambda(auto:1)#1}&&, boost::intrusive_ptr<crimson::os::seastore::CachedExtent>&&) at /home/xuxuehan/nvme/ceph/build/../src/seastar/include/seastar/core/shared_ptr.hh:300
Actions #1

Updated by Xuehan Xu almost 2 years ago

  • Pull request ID set to 47028
Actions #2

Updated by Xuehan Xu over 1 year ago

  • Status changed from New to Resolved
Actions

Also available in: Atom PDF