Ceph » crimson

Journal triming's target is two segment seq before the current journal_head. As we use non-continuous journal segment seq, that target may well be only one record before journal_head

DEBUG 2022-03-30 19:09:59,557 [shard 0] seastore_cache - 0x5115450 Cache::complete_commit: add extent as fresh, inline=true -- CachedExtent(addr=0x1ac26100, type=OBJECT_DATA_BLOCK, version=0, dirty_from_or_retired_at=JOURNAL_SEQ_NULL, last_modified=1648638720.000000000s, last_rewritten=0.000000000s, paddr=paddr_t<[0,48], 8192>, length=4096, state=CLEAN, last_committed_crc=808552097, refcount=1, laddr=18375156437720682496, pin=LBAPin(18375156437720682496~4096->paddr_t<RECORD_REG, 0>)
DEBUG 2022-03-30 19:09:59,557 [shard 0] seastore_cache - 0x5115450 Cache::complete_commit: commit extent done -- CachedExtent(addr=0x6cdab10, type=ONODE_BLOCK_STAGED, version=9, dirty_from_or_retired_at=journal_seq_t(segment_seq=40, offset=paddr_t<[0,46], 12795904>), last_modified=1648638720.000000000s, last_rewritten=0.000000000s, paddr=paddr_t<[0,47], 5025792>, length=16384, state=DIRTY, last_committed_crc=488547772, refcount=2, laddr=18375156695907893248, pin=LBAPin(18375156695907893248~16384->paddr_t<RECORD_REG, 0>, fltree_header=headerL0(is_level_tail=0, level=0))
DEBUG 2022-03-30 19:09:59,557 [shard 0] seastore_cache - 0x5115450 Cache::complete_commit: commit extent done -- CachedExtent(addr=0xa5e54e0, type=LADDR_LEAF, version=3, dirty_from_or_retired_at=journal_seq_t(segment_seq=40, offset=paddr_t<[0,46], 18972672>), last_modified=1648638720.000000000s, last_rewritten=0.000000000s, paddr=paddr_t<[0,45], 64966656>, length=4096, state=DIRTY, last_committed_crc=477802431, refcount=4, size=82, meta=btree_node_meta_t(begin=18375156419775688704, end=18375156462111567872, depth=1))
DEBUG 2022-03-30 19:09:59,557 [shard 0] seastore_cache - 0x5115450 Cache::complete_commit: commit extent done -- CachedExtent(addr=0x112f31a0, type=ONODE_BLOCK_STAGED, version=623, dirty_from_or_retired_at=journal_seq_t(segment_seq=40, offset=paddr_t<[0,46], 12611584>), last_modified=1648638720.000000000s, last_rewritten=0.000000000s, paddr=paddr_t<[0,28], 61984768>, length=16384, state=DIRTY, last_committed_crc=397854456, refcount=2, laddr=18375144154596114432, pin=LBAPin(18375144154596114432~16384->paddr_t<RECORD_REG, 0>, fltree_header=headerL0(is_level_tail=0, level=0))
DEBUG 2022-03-30 19:09:59,557 [shard 0] seastore_cache - 0x5115450 Cache::complete_commit: commit extent done -- CachedExtent(addr=0x136ea130, type=OMAP_LEAF, version=4, dirty_from_or_retired_at=journal_seq_t(segment_seq=40, offset=paddr_t<[0,46], 66887680>), last_modified=1648638720.000000000s, last_rewritten=0.000000000s, paddr=paddr_t<[0,46], 66818048>, length=4096, state=DIRTY, last_committed_crc=2620169045, refcount=2, laddr=18375143876540436480, pin=LBAPin(18375143876540436480~4096->paddr_t<RECORD_REG, 8192>, size=13, depth=1)
DEBUG 2022-03-30 19:09:59,557 [shard 0] seastore_lba - 0x5115450 BtreeLBAManager::complete_transaction: start
DEBUG 2022-03-30 19:09:59,557 [shard 0] seastore_lba - 0x5115450 BtreeLBAManager::complete_transaction: linking extent -- CachedExtent(addr=0x1ac26100, type=OBJECT_DATA_BLOCK, version=0, dirty_from_or_retired_at=JOURNAL_SEQ_NULL, last_modified=1648638720.000000000s, last_rewritten=0.000000000s, paddr=paddr_t<[0,48], 8192>, length=4096, state=CLEAN, last_committed_crc=808552097, refcount=3, laddr=18375156437720682496, pin=LBAPin(18375156437720682496~4096->paddr_t<RECORD_REG, 0>)
DEBUG 2022-03-30 19:09:59,557 [shard 0] seastore_lba - add_pin: parent has ref 0xa5e55c8
DEBUG 2022-03-30 19:09:59,557 [shard 0] seastore_cleaner - update_journal_tail_target: journal_seq_t(segment_seq=39, offset=paddr_t<[0,45], 28672>), current tail target journal_seq_t(segment_seq=39, offset=paddr_t<[0,45], 28672>)
DEBUG 2022-03-30 19:09:59,557 [shard 0] seastore_lba_details - 0xd393e20 FixedKVBtree::get_leaf_node: read leaf at offset paddr_t<[0,46], 10653696> CachedExtent(addr=0x5f2e820, type=LADDR_LEAF, version=4, dirty_from_or_retired_at=journal_seq_t(segment_seq=40, offset=paddr_t<[0,46], 14962688>), last_modified=1648638720.000000000s, last_rewritten=0.000000000s, paddr=paddr_t<[0,46], 10653696>, length=4096, state=DIRTY, last_committed_crc=3122860596, refcount=6, size=91, meta=btree_node_meta_t(begin=18375223009796366336, end=18375223037690052608, depth=1))
DEBUG 2022-03-30 19:09:59,557 [shard 0] seastore_lba_details - 0xd174a10 FixedKVBtree::get_internal_node: reading internal at offset paddr_t<[0,43], 1159168>, depth 3, begin 0, end 18446744073709551615
DEBUG 2022-03-30 19:09:59,557 [shard 0] seastore_lba_details - 0x6b66b60 FixedKVBtree::lookup: got root, depth 3
DEBUG 2022-03-30 19:09:59,557 [shard 0] seastore_lba_details - 0x6b66b60 FixedKVBtree::lookup_depth_range: 2 -> 0
DEBUG 2022-03-30 19:09:59,557 [shard 0] seastore_lba_details - 0x6b66b60 FixedKVBtree::get_internal_node: reading internal at offset paddr_t<[0,46], 9113600>, depth 2, begin 18375101605849419776, end 18375120233092689920
DEBUG 2022-03-30 19:09:59,557 [shard 0] seastore_cleaner - SegmentCleaner::log_gc_state(GCProcess::maybe_wait_should_run): total 53687091200, available 50593017856, unavailable 3094073344, used 793866240, reclaimable 2031771648, reclaim_ratio 0.6566656384988396, available_ratio 0.9423683929443359, should_block_on_gc false, gc_should_reclaim_space false, journal_head journal_seq_t(segment_seq=42, offset=paddr_t<[0,48], 12288>), journal_tail_target journal_seq_t(segment_seq=39, offset=paddr_t<[0,45], 28672>), journal_tail_commit journal_seq_t(segment_seq=39, offset=paddr_t<[0,45], 28672>), dirty_tail journal_seq_t(segment_seq=40, offset=paddr_t<[0,48], 12288>), dirty_tail_limit journal_seq_t(segment_seq=38, offset=paddr_t<[0,48], 12288>), gc_should_trim_journal true,
DEBUG 2022-03-30 19:09:59,557 [shard 0] seastore_lba_details - 0x6b66b60 FixedKVBtree::get_internal_node: read internal at offset paddr_t<[0,46], 9113600> CachedExtent(addr=0x13d2e8c0, type=LADDR_INTERNAL, version=14, dirty_from_or_retired_at=journal_seq_t(segment_seq=40, offset=paddr_t<[0,46], 19259392>), last_modified=1648638720.000000000s, last_rewritten=0.000000000s, paddr=paddr_t<[0,46], 9113600>, length=4096, state=DIRTY, last_committed_crc=1073523886, refcount=8, size=247, meta=btree_node_meta_t(begin=18375101605849419776, end=18375120233092689920, depth=2))
DEBUG 2022-03-30 19:09:59,557 [shard 0] seastore_lba_details - 0x63fda60 FixedKVBtree::get_leaf_node: read leaf at offset paddr_t<[0,46], 6078464> CachedExtent(addr=0xc41a950, type=LADDR_LEAF, version=2, dirty_from_or_retired_at=journal_seq_t(segment_seq=40, offset=paddr_t<[0,46], 33247232>), last_modified=1648638720.000000000s, last_rewritten=0.000000000s, paddr=paddr_t<[0,46], 6078464>, length=4096, state=DIRTY, last_committed_crc=1539834761, refcount=5, size=137, meta=btree_node_meta_t(begin=18375104138278612992, end=18375104334454910976, depth=1))
DEBUG 2022-03-30 19:09:59,557 [shard 0] seastore_lba_details - 0x63fda60 FixedKVBtree::lower_bound: leaf addr 18375104175483453440, got ret offset 59, size 137, end false
DEBUG 2022-03-30 19:09:59,557 [shard 0] seastore_lba_details - 0xd393e20 FixedKVBtree::lower_bound: leaf addr 18375223013766705152, got ret offset 31, size 91, end false
DEBUG 2022-03-30 19:09:59,557 [shard 0] seastore_cleaner - SegmentCleaner::log_gc_state(GCProcess::run): total 53687091200, available 50593017856, unavailable 3094073344, used 793866240, reclaimable 2031771648, reclaim_ratio 0.6566656384988396, available_ratio 0.9423683929443359, should_block_on_gc false, gc_should_reclaim_space false, journal_head journal_seq_t(segment_seq=42, offset=paddr_t<[0,48], 12288>), journal_tail_target journal_seq_t(segment_seq=39, offset=paddr_t<[0,45], 28672>), journal_tail_commit journal_seq_t(segment_seq=39, offset=paddr_t<[0,45], 28672>), dirty_tail journal_seq_t(segment_seq=40, offset=paddr_t<[0,48], 12288>), dirty_tail_limit journal_seq_t(segment_seq=38, offset=paddr_t<[0,48], 12288>), gc_should_trim_journal true,
DEBUG 2022-03-30 19:09:59,557 [shard 0] seastore_t - 0x13855a10 Cache::create_transaction: created name=trim_journal, source=CLEANER_TRIM, is_weak=false
DEBUG 2022-03-30 19:09:59,557 [shard 0] seastore_tm - 0x13855a10 TransactionManager::get_next_dirty_extents: max_bytes=33554432B, seq=journal_seq_t(segment_seq=40, offset=paddr_t<[0,48], 12288>)
DEBUG 2022-03-30 19:09:59,557 [shard 0] seastore_cache - 0x13855a10 Cache::get_next_dirty_extents: max_bytes=33554432B, seq=journal_seq_t(segment_seq=40, offset=paddr_t<[0,48], 12288>), dirty_from=journal_seq_t(segment_seq=39, offset=paddr_t<[0,45], 28672>)
ERROR 2022-03-30 19:09:59,557 [shard 0] seastore_cache - 0x13855a10 Cache::get_next_dirty_extents: CachedExtent(addr=0xa8c3910, type=LADDR_LEAF, version=1, dirty_from_or_retired_at=JOURNAL_SEQ_NULL, last_modified=1648638720.000000000s, last_rewritten=0.000000000s, paddr=paddr_t<[0,46], 10051584>, length=4096, state=MUTATION_PENDING, last_committed_crc=4270638727, refcount=3, size=91, meta=btree_node_meta_t(begin=18375186188542083072, end=18375186263430725632, depth=1))
ERROR 2022-03-30 19:09:59,558 [shard 0] none - ../src/crimson/os/seastore/cache.cc:1396 : In function 'crimson::os::seastore::Cache::get_next_dirty_extents_ret crimson::os::seastore::Cache::get_next_dirty_extents(crimson::os::seastore::Transaction&, crimson::os::seastore::journal_seq_t, size_t)', ceph_assert(%s)
dirty_from != JOURNAL_SEQ_NULL && dirty_from != JOURNAL_SEQ_MAX && dirty_from != NO_DELTAS
Aborting on shard 0.
Backtrace:
 0# gsignal in /lib64/libc.so.6
 1# abort in /lib64/libc.so.6
 2# ceph::__ceph_assert_fail(char const*, char const*, int, char const*) at /home/xuxuehan/src/ceph/build/../src/seastar/include/seastar/util/log.hh:106
 3# ceph::__ceph_assertf_fail(char const*, char const*, int, char const*, char const*, ...) at /home/xuxuehan/src/ceph/build/../src/crimson/common/assert.cc:33
 4# crimson::os::seastore::Cache::get_next_dirty_extents(crimson::os::seastore::Transaction&, crimson::os::seastore::journal_seq_t, unsigned long) at /home/xuxuehan/src/ceph/build/../src/crimson/os/seastore/cache.cc:1401
 5# crimson::os::seastore::TransactionManager::get_next_dirty_extents(crimson::os::seastore::Transaction&, crimson::os::seastore::journal_seq_t, unsigned long) at /home/xuxuehan/src/ceph/build/../src/crimson/os/seastore/transaction_manager.cc:395
 6# crimson::os::seastore::SegmentCleaner::rewrite_dirty(crimson::os::seastore::Transaction&, crimson::os::seastore::journal_seq_t) at /home/xuxuehan/src/ceph/build/../src/crimson/os/seastore/segment_cleaner.cc:324
 7# operator()<crimson::os::seastore::Transaction> at /home/xuxuehan/src/ceph/build/../src/crimson/os/seastore/segment_cleaner.cc:372
10# auto crimson::os::seastore::with_trans_intr<crimson::os::seastore::SegmentCleaner::ExtentCallbackInterface::with_transaction_intr<crimson::os::seastore::SegmentCleaner::gc_trim_journal()::{lambda()#1}::operator()() const::{lambda(auto:1&)#1}>(crimson::os::seastore::Transaction::src_t, char const*, {lambda()#1}&&)::{lambda(auto:1)#1}::operator()<std::unique_ptr<{lambda(auto:1&)#1}, std::default_delete<{lambda(auto:1&)#1}> > >(std::unique_ptr<{lambda(auto:1&)#1}, std::default_delete<{lambda(auto:1&)#1}> >)::{lambda(auto:1)#1}>({lambda(auto:1&)#1}&, char const) at /home/xuxuehan/src/ceph/build/../src/crimson/os/seastore/transaction.h:520
11# operator()<std::unique_ptr<crimson::os::seastore::Transaction> > at /home/xuxuehan/src/ceph/build/../src/crimson/os/seastore/segment_cleaner.h:589
12# auto seastar::futurize_invoke<crimson::errorator<crimson::unthrowable_wrapper<std::error_code const&, crimson::ec<(std::errc)5> >, crimson::unthrowable_wrapper<std::error_code const&, crimson::ec<(std::errc)11> > >::_future<crimson::errorated_future_marker<void> > (*&)(std::unique_ptr<crimson::os::seastore::Transaction, std::default_delete<crimson::os::seastore::Transaction> >&&, crimson::os::seastore::SegmentCleaner::ExtentCallbackInterface::with_transaction_intr<crimson::os::seastore::SegmentCleaner::gc_trim_journal()::{lambda()#1}::operator()() const::{lambda(auto:1&)#1}>(crimson::os::seastore::Transaction::src_t, char const*, {lambda()#1}&&)::{lambda(auto:1)#1}&&), std::unique_ptr<crimson::os::seastore::Transaction, std::default_delete<crimson::os::seastore::Transaction> >, char const*>(char const, std::unique_ptr<crimson::os::seastore::Transaction, std::default_delete<crimson::os::seastore::Transaction> >&&, char const*&&) at /home/xuxuehan/src/ceph/build/../src/crimson/common/errorator.h:1222
13# crimson::os::seastore::repeat_eagain<crimson::os::seastore::SegmentCleaner::gc_trim_journal()::{lambda()#1}>(crimson::os::seastore::SegmentCleaner::gc_trim_journal()::{lambda()#1}&&)::{lambda(crimson::os::seastore::SegmentCleaner::gc_trim_journal()::{lambda()#1}&)#1}::operator()<{lambda()#1}>(crimson::os::seastore::SegmentCleaner::gc_trim_journal()::{lambda()#1}&&) const::{lambda()#1}::operator()() const at /opt/rh/gcc-toolset-11/root/usr/include/c++/11/bits/unique_ptr.h:360
14# auto seastar::futurize_invoke<crimson::os::seastore::repeat_eagain<crimson::os::seastore::SegmentCleaner::gc_trim_journal()::{lambda()#1}>(crimson::os::seastore::SegmentCleaner::gc_trim_journal()::{lambda()#1}&&)::{lambda(crimson::os::seastore::SegmentCleaner::gc_trim_journal()::{lambda()#1}&)#1}::operator()<{lambda()#1}>(crimson::os::seastore::SegmentCleaner::gc_trim_journal()::{lambda()#1}&&) const::{lambda()#1}&>(crimson::os::seastore::repeat_eagain<crimson::os::seastore::SegmentCleaner::gc_trim_journal()::{lambda()#1}>(crimson::os::seastore::SegmentCleaner::gc_trim_journal()::{lambda()#1}&&)::{lambda(crimson::os::seastore::SegmentCleaner::gc_trim_journal()::{lambda()#1}&)#1}::operator()<{lambda()#1}>({lambda()#1}&&) const::{lambda()#1}&) at /home/xuxuehan/src/ceph/build/../src/crimson/common/errorator.h:1222
15# auto crimson::repeat<crimson::os::seastore::repeat_eagain<crimson::os::seastore::SegmentCleaner::gc_trim_journal()::{lambda()#1}>(crimson::os::seastore::SegmentCleaner::gc_trim_journal()::{lambda()#1}&&)::{lambda(crimson::os::seastore::SegmentCleaner::gc_trim_journal()::{lambda()#1}&)#1}::operator()<{lambda()#1}>(crimson::os::seastore::SegmentCleaner::gc_trim_journal()::{lambda()#1}&&) const::{lambda()#1}>(crimson::os::seastore::SegmentCleaner::gc_trim_journal()::{lambda()#1}) at /home/xuxuehan/src/ceph/build/../src/seastar/include/seastar/core/future.hh:454
16# auto seastar::internal::do_with_impl<crimson::os::seastore::SegmentCleaner::gc_trim_journal()::{lambda()#1}, crimson::os::seastore::repeat_eagain<{lambda()#1}>({lambda()#1}&&)::{lambda(crimson::os::seastore::repeat_eagain&)#1}>(crimson::os::seastore::SegmentCleaner::gc_trim_journal()::{lambda()#1}, crimson::os::seastore::repeat_eagain<{lambda()#1}>({lambda()#1}&&)::{lambda(crimson::os::seastore::repeat_eagain&)#1}&&) at /home/xuxuehan/src/ceph/build/../src/seastar/include/seastar/core/do_with.hh:97
17# auto seastar::futurize_invoke<crimson::errorator<crimson::unthrowable_wrapper<std::error_code const&, crimson::ec<(std::errc)5> > >::_future<crimson::errorated_future_marker<void> > (*&)(crimson::os::seastore::SegmentCleaner::gc_trim_journal()::{lambda()#1}&&, crimson::os::seastore::repeat_eagain<{lambda()#1}>({lambda()#1}&&)::{lambda(crimson::os::seastore::repeat_eagain&)#1}&&), {lambda()#1}, crimson::os::seastore::repeat_eagain&>(crimson::errorator<crimson::unthrowable_wrapper<std::error_code const&, crimson::ec<(std::errc)5> > >::_future<crimson::errorated_future_marker<void> > (*&)(crimson::os::seastore::SegmentCleaner::gc_trim_journal()::{lambda()#1}&&, crimson::os::seastore::repeat_eagain<{lambda()#1}>({lambda()#1}&&)::{lambda(crimson::os::seastore::repeat_eagain&)#1}&&), {lambda()#1}&&, crimson::os::seastore::repeat_eagain&) at /home/xuxuehan/src/ceph/build/../src/crimson/common/errorator.h:1222
18# auto crimson::os::seastore::repeat_eagain<crimson::os::seastore::SegmentCleaner::gc_trim_journal()::{lambda()#1}>(crimson::os::seastore::SegmentCleaner::gc_trim_journal()::{lambda()#1}&&) at /home/xuxuehan/src/ceph/build/../src/crimson/os/seastore/transaction_manager.h:53
19# crimson::os::seastore::SegmentCleaner::gc_trim_journal() at /home/xuxuehan/src/ceph/build/../src/seastar/include/seastar/core/future.hh:1146
20# crimson::os::seastore::SegmentCleaner::do_gc_cycle() at /home/xuxuehan/src/ceph/build/../src/crimson/common/errorator.h:713
21# operator() at /home/xuxuehan/src/ceph/build/../src/crimson/os/seastore/segment_cleaner.cc:339
22# _ZN7seastar20noncopyable_functionIFNS_6futureIvEEvEE17direct_vtable_forIZNS2_4thenIZZN7crimson2os8seastore14SegmentCleaner9GCProcess3runEvENKUlvE0_clEvEUlvE_S2_EET0_OT_EUlDpOT_E_E4callEPKS4_ at /home/xuxuehan/src/ceph/build/../src/seastar/include/seastar/util/noncopyable_function.hh:125
23# auto seastar::internal::future_invoke<seastar::noncopyable_function<seastar::future<void> ()>&, seastar::internal::monostate>(seastar::noncopyable_function<seastar::future<void> ()>&, seastar::internal::monostate&&) at /home/xuxuehan/src/ceph/build/../src/seastar/include/seastar/core/future.hh:1213
24# void seastar::futurize<seastar::future<void> >::satisfy_with_result_of<seastar::future<void>::then_impl_nrvo<seastar::noncopyable_function<seastar::future<void> ()>, seastar::future<void> >(seastar::noncopyable_function<seastar::future<void> ()>&&)::{lambda(seastar::internal::promise_base_with_type<void>&&, seastar::noncopyable_function<seastar::future<void> ()>&, seastar::future_state<seastar::internal::monostate>&&)#1}::operator()(seastar::internal::promise_base_with_type<void>&&, seastar::noncopyable_function<seastar::future<void> ()>&, seastar::future_state<seastar::internal::monostate>&&) const::{lambda()#1}>(seastar::internal::promise_base_with_type<void>&&, seastar::noncopyable_function<seastar::future<void> ()>&&) at /home/xuxuehan/src/ceph/build/../src/seastar/include/seastar/core/future.hh:1695
25# seastar::future<void>::then_impl_nrvo<seastar::noncopyable_function<seastar::future<void> ()>, seastar::future<void> >(seastar::noncopyable_function<seastar::future<void> ()>&&)::{lambda(seastar::internal::promise_base_with_type<void>&&, seastar::noncopyable_function<seastar::future<void> ()>&, seastar::future_state<seastar::internal::monostate>&&)#1}::operator()(seastar::internal::promise_base_with_type<void>&&, seastar::noncopyable_function<seastar::future<void> ()>&, seastar::future_state<seastar::internal::monostate>&&) const at /home/xuxuehan/src/ceph/build/../src/seastar/include/seastar/core/future.hh:1575
26# seastar::reactor::run_tasks(seastar::reactor::task_queue&) at /home/xuxuehan/src/ceph/build/../src/seastar/src/core/reactor.cc:2345
27# seastar::reactor::run_some_tasks() at /home/xuxuehan/src/ceph/build/../src/seastar/src/core/reactor.cc:2755
28# seastar::reactor::do_run() at /home/xuxuehan/src/ceph/build/../src/seastar/src/core/reactor.cc:2923
29# seastar::reactor::run() at /home/xuxuehan/src/ceph/build/../src/seastar/src/core/reactor.cc:2806
30# seastar::app_template::run_deprecated(int, char**, std::function<void ()>&&) at /home/xuxuehan/src/ceph/build/../src/seastar/src/core/app-template.cc:265
31# seastar::app_template::run(int, char**, std::function<seastar::future<int> ()>&&) at /home/xuxuehan/src/ceph/build/../src/seastar/src/core/app-template.cc:156
32# main at /home/xuxuehan/src/ceph/build/../src/crimson/osd/main.cc:238
33# __libc_start_main in /lib64/libc.so.6
34# _start in /home/xuxuehan/src/ceph/build/bin/crimson-osd