Ceph » crimson

c49b81c7d619cea23e9707d1f5bcc7de3049c4fd with the folowing debugging (may have changed the continuation timing)

diff --git a/src/crimson/osd/osd_operations/client_request.cc b/src/crimson/osd/osd_operations/client_request.cc
index db531f7bcb2..6bfa71d2e28 100644
--- a/src/crimson/osd/osd_operations/client_request.cc
+++ b/src/crimson/osd/osd_operations/client_request.cc
@@ -129,7 +129,7 @@ seastar::future<> ClientRequest::with_pg_int(
       }
       return enter_stage<interruptor>(pp(pg).await_map
       ).then_interruptible([this, this_instance_id, &pg] {
-       logger().debug("{}.{}: after await_map stage", *this, this_instance_id);
+       logger().debug("{}.{}: in await_map stage", *this, this_instance_id);
        return with_blocking_event<
          PG_OSDMapGate::OSDMapBlocker::BlockingEvent
          >([this, &pg] (auto&& trigger) {
@@ -138,20 +138,22 @@ seastar::future<> ClientRequest::with_pg_int(
              m->get_min_epoch());
          });
       }).then_interruptible([this, this_instance_id, &pg](auto map) {
-       logger().debug("{}.{}: after wait_for_map", *this, this_instance_id);
+       logger().debug("{}.{}: have map", *this, this_instance_id);
        return enter_stage<interruptor>(pp(pg).wait_for_active);
       }).then_interruptible([this, this_instance_id, &pg]() {
        logger().debug(
-         "{}.{}: after wait_for_active stage", *this, this_instance_id);
+         "{}.{}: in wait_for_active stage", *this, this_instance_id);
        return with_blocking_event<
          PGActivationBlocker::BlockingEvent
-         >([&pg] (auto&& trigger) {
+         >([this, this_instance_id, &pg] (auto&& trigger) {
+           logger().debug(
+             "{}.{}: awaiting active {}", *this, this_instance_id, pg);
            return pg.wait_for_active_blocker.wait(std::move(trigger));
          });
       }).then_interruptible([this, pgref, this_instance_id]() mutable
                            -> interruptible_future<> {
        logger().debug(
-         "{}.{}: after wait_for_active", *this, this_instance_id);
+         "{}.{}: active, executing op", *this, this_instance_id);
        if (is_pg_op()) {
          return process_pg_op(pgref);
        } else {

home/sam/git-checkouts/ceph2/src/ceph-run --no-restart /home/sam/git-checkouts/ceph2/build/bin/crimson-osd --smp 1 --cpuset 1 --debug -i 1 -c /home/sam/git-checkouts/ceph2/build/ceph.conf -f & 
WARNING: debug mode. Not for benchmarking or production
WARN  2022-09-10 22:52:04,861 [shard 0] seastar - Creation of perf_event based stall detector failed, falling back to posix timer: std::system_error (error system:13, perf_event_open() failed: Permission denied)
INFO  2022-09-10 22:52:04,870 [shard 0] seastar - Created fair group io-queue-0, capacity rate 2147483:2147483, limit 12582912, rate 16777216 (factor 1), threshold 2000
INFO  2022-09-10 22:52:04,870 [shard 0] seastar - Created io group dev(0), length limit 4194304:4194304, rate 2147483647:2147483647
INFO  2022-09-10 22:52:04,870 [shard 0] seastar - Created io queue dev(0) capacities: 512:2000/2000 1024:3000/3000 2048:5000/5000 4096:9000/9000 8192:17000/17000 16384:33000/33000 32768:65000/65000 65536:129000/129000 131072:257000/257000
==436905==WARNING: ASan doesn't fully support makecontext/swapcontext functions and may produce false positives in some cases!
==436905==WARNING: ASan is ignoring requested __asan_handle_no_return: stack type: default top: 0x7ffe2b811000; bottom 0x7f8178690000; size: 0x007cb3181000 (535580643328)
False positive error reports may follow
For details see https://github.com/google/sanitizers/issues/189
crimson-osd: ../src/crimson/common/operation.h:183: static void crimson::BlockerT<T>::BlockingEvent::TriggerI::record_unblocking(crimson::BlockerT<T>::BlockingEvent&, const T&) [with T = crimson::osd::CommonPGPipeline::WaitForActive]: Assertion `event.internal_backend.blocker == &blocker' failed.
Aborting on shard 0.
Backtrace:
Reactor stalled for 10152 ms on shard 0. Backtrace: 0x44700 0xeec50f5 0xec5e9a9 0xec7c265 0xec7c708 0xec7c998 0xec7cc63 0x7f818882da1f 0xccd78 0x7436f46 0x743a9be 0x743e51d 0x743f17e 0x743f848 0x7433673 0x7433b4f 0x74340ec 0x7f818882da1f 0x3d2a1 0x268a3 0x26788 0x35a15 0x3525ec4 0x2193743 0x219654e 0xec3aace 0xec973ca 0xedfed84 0xee00cf4 0xe837c4f 0xe83c5b9 0x1a9162f 0x27b74 0x175a58d
 0# gsignal in /lib64/libc.so.6
 1# abort in /lib64/libc.so.6
 2# 0x00007F8187981789 in /lib64/libc.so.6
 3# 0x00007F8187990A16 in /lib64/libc.so.6
 4# seastar::noncopyable_function<seastar::future<std::unique_ptr<crimson::PipelineExitBarrierI, std::default_delete<crimson::PipelineExitBarrierI> > > (seastar::future<std::unique_ptr<crimson::PipelineExitBarrierI, std::default_delete<crimson::PipelineExitBarrierI> > >&&)>::direct_vtable_for<seastar::future<std::unique_ptr<crimson::PipelineExitBarrierI, std::default_delete<crimson::PipelineExitBarrierI> > >::then_wrapped_maybe_erase<false, seastar::future<std::unique_ptr<crimson::PipelineExitBarrierI, std::default_delete<crimson::PipelineExitBarrierI> > >, seastar::future<std::unique_ptr<crimson::PipelineExitBarrierI, std::default_delete<crimson::PipelineExitBarrierI> > >::finally_body<crimson::BlockerT<crimson::osd::CommonPGPipeline::WaitForActive>::BlockingEvent::Trigger<crimson::osd::ClientRequest>::maybe_record_blocking<seastar::future<std::unique_ptr<crimson::PipelineExitBarrierI, std::default_delete<crimson::PipelineExitBarrierI> > > >(seastar::future<std::unique_ptr<crimson::PipelineExitBarrierI, std::default_delete<crimson::PipelineExitBarrierI> > >&&, crimson::osd::CommonPGPipeline::WaitForActive const&)::{lambda()#1}, false> >(seastar::future<std::unique_ptr<crimson::PipelineExitBarrierI, std::default_delete<crimson::PipelineExitBarrierI> > >::finally_body<crimson::BlockerT<crimson::osd::CommonPGPipeline::WaitForActive>::BlockingEvent::Trigger<crimson::osd::ClientRequest>::maybe_record_blocking<seastar::future<std::unique_ptr<crimson::PipelineExitBarrierI, std::default_delete<crimson::PipelineExitBarrierI> > > >(seastar::future<std::unique_ptr<crimson::PipelineExitBarrierI, std::default_delete<crimson::PipelineExitBarrierI> > >&&, crimson::osd::CommonPGPipeline::WaitForActive const&)::{lambda()#1}, false>&&)::{lambda(seastar::future<std::unique_ptr<crimson::PipelineExitBarrierI, std::default_delete<crimson::PipelineExitBarrierI> > >&&)#1}>::call(seastar::noncopyable_function<seastar::future<std::unique_ptr<crimson::PipelineExitBarrierI, std::default_delete<crimson::PipelineExitBarrierI> > > (seastar::future<std::unique_ptr<crimson::PipelineExitBarrierI, std::default_delete<crimson::PipelineExitBarrierI> > >&&)> const*, seastar::future<std::unique_ptr<crimson::PipelineExitBarrierI, std::default_delete<crimson::PipelineExitBarrierI> > >&&) at /home/sam/git-checkouts/ceph2/build/../src/crimson/common/operation.h:184
 5# void seastar::futurize<seastar::future<std::unique_ptr<crimson::PipelineExitBarrierI, std::default_delete<crimson::PipelineExitBarrierI> > > >::satisfy_with_result_of<seastar::future<std::unique_ptr<crimson::PipelineExitBarrierI, std::default_delete<crimson::PipelineExitBarrierI> > >::then_wrapped_nrvo<seastar::future<std::unique_ptr<crimson::PipelineExitBarrierI, std::default_delete<crimson::PipelineExitBarrierI> > >, seastar::noncopyable_function<seastar::future<std::unique_ptr<crimson::PipelineExitBarrierI, std::default_delete<crimson::PipelineExitBarrierI> > > (seastar::future<std::unique_ptr<crimson::PipelineExitBarrierI, std::default_delete<crimson::PipelineExitBarrierI> > >&&)> >(seastar::noncopyable_function<seastar::future<std::unique_ptr<crimson::PipelineExitBarrierI, std::default_delete<crimson::PipelineExitBarrierI> > > (seastar::future<std::unique_ptr<crimson::PipelineExitBarrierI, std::default_delete<crimson::PipelineExitBarrierI> > >&&)>&&)::{lambda(seastar::internal::promise_base_with_type<std::unique_ptr<crimson::PipelineExitBarrierI, std::default_delete<crimson::PipelineExitBarrierI> > >&&, seastar::noncopyable_function<seastar::future<std::unique_ptr<crimson::PipelineExitBarrierI, std::default_delete<crimson::PipelineExitBarrierI> > > (seastar::future<std::unique_ptr<crimson::PipelineExitBarrierI, std::default_delete<crimson::PipelineExitBarrierI> > >&&)>&, seastar::future_state<std::unique_ptr<crimson::PipelineExitBarrierI, std::default_delete<crimson::PipelineExitBarrierI> > >&&)#1}::operator()(seastar::internal::promise_base_with_type<std::unique_ptr<crimson::PipelineExitBarrierI, std::default_delete<crimson::PipelineExitBarrierI> > >&&, seastar::noncopyable_function<seastar::future<std::unique_ptr<crimson::PipelineExitBarrierI, std::default_delete<crimson::PipelineExitBarrierI> > > (seastar::future<std::unique_ptr<crimson::PipelineExitBarrierI, std::default_delete<crimson::PipelineExitBarrierI> > >&&)>&, seastar::future_state<std::unique_ptr<crimson::PipelineExitBarrierI, std::default_delete<crimson::PipelineExitBarrierI> > >&&) const::{lambda()#1}>(seastar::internal::promise_base_with_type<std::unique_ptr<crimson::PipelineExitBarrierI, std::default_delete<crimson::PipelineExitBarrierI> > >&&, seastar::future<std::unique_ptr<crimson::PipelineExitBarrierI, std::default_delete<crimson::PipelineExitBarrierI> > >::then_wrapped_nrvo<seastar::future<std::unique_ptr<crimson::PipelineExitBarrierI, std::default_delete<crimson::PipelineExitBarrierI> > >, seastar::noncopyable_function<seastar::future<std::unique_ptr<crimson::PipelineExitBarrierI, std::default_delete<crimson::PipelineExitBarrierI> > > (seastar::future<std::unique_ptr<crimson::PipelineExitBarrierI, std::default_delete<crimson::PipelineExitBarrierI> > >&&)> >(seastar::noncopyable_function<seastar::future<std::unique_ptr<crimson::PipelineExitBarrierI, std::default_delete<crimson::PipelineExitBarrierI> > > (seastar::future<std::unique_ptr<crimson::PipelineExitBarrierI, std::default_delete<crimson::PipelineExitBarrierI> > >&&)>&&)::{lambda(seastar::internal::promise_base_with_type<std::unique_ptr<crimson::PipelineExitBarrierI, std::default_delete<crimson::PipelineExitBarrierI> > >&&, seastar::noncopyable_function<seastar::future<std::unique_ptr<crimson::PipelineExitBarrierI, std::default_delete<crimson::PipelineExitBarrierI> > > (seastar::future<std::unique_ptr<crimson::PipelineExitBarrierI, std::default_delete<crimson::PipelineExitBarrierI> > >&&)>&, seastar::future_state<std::unique_ptr<crimson::PipelineExitBarrierI, std::default_delete<crimson::PipelineExitBarrierI> > >&&)#1}::operator()(seastar::internal::promise_base_with_type<std::unique_ptr<crimson::PipelineExitBarrierI, std::default_delete<crimson::PipelineExitBarrierI> > >&&, seastar::noncopyable_function<seastar::future<std::unique_ptr<crimson::PipelineExitBarrierI, std::default_delete<crimson::PipelineExitBarrierI> > > (seastar::future<std::unique_ptr<crimson::PipelineExitBarrierI, std::default_delete<crimson::PipelineExitBarrierI> > >&&)>&, seastar::future_state<std::unique_ptr<crimson::PipelineExitBarrierI, std::default_delete<crimson::PipelineExitBarrierI> > >&&) const::{lambda()#1}&&) at /home/sam/git-checkouts/ceph2/build/../src/seastar/include/seastar/util/noncopyable_function.hh:209
 6# seastar::continuation<seastar::internal::promise_base_with_type<std::unique_ptr<crimson::PipelineExitBarrierI, std::default_delete<crimson::PipelineExitBarrierI> > >, seastar::noncopyable_function<seastar::future<std::unique_ptr<crimson::PipelineExitBarrierI, std::default_delete<crimson::PipelineExitBarrierI> > > (seastar::future<std::unique_ptr<crimson::PipelineExitBarrierI, std::default_delete<crimson::PipelineExitBarrierI> > >&&)>, seastar::future<std::unique_ptr<crimson::PipelineExitBarrierI, std::default_delete<crimson::PipelineExitBarrierI> > >::then_wrapped_nrvo<seastar::future<std::unique_ptr<crimson::PipelineExitBarrierI, std::default_delete<crimson::PipelineExitBarrierI> > >, seastar::noncopyable_function<seastar::future<std::unique_ptr<crimson::PipelineExitBarrierI, std::default_delete<crimson::PipelineExitBarrierI> > > (seastar::future<std::unique_ptr<crimson::PipelineExitBarrierI, std::default_delete<crimson::PipelineExitBarrierI> > >&&)> >(seastar::noncopyable_function<seastar::future<std::unique_ptr<crimson::PipelineExitBarrierI, std::default_delete<crimson::PipelineExitBarrierI> > > (seastar::future<std::unique_ptr<crimson::PipelineExitBarrierI, std::default_delete<crimson::PipelineExitBarrierI> > >&&)>&&)::{lambda(seastar::internal::promise_base_with_type<std::unique_ptr<crimson::PipelineExitBarrierI, std::default_delete<crimson::PipelineExitBarrierI> > >&&, seastar::noncopyable_function<seastar::future<std::unique_ptr<crimson::PipelineExitBarrierI, std::default_delete<crimson::PipelineExitBarrierI> > > (seastar::future<std::unique_ptr<crimson::PipelineExitBarrierI, std::default_delete<crimson::PipelineExitBarrierI> > >&&)>&, seastar::future_state<std::unique_ptr<crimson::PipelineExitBarrierI, std::default_delete<crimson::PipelineExitBarrierI> > >&&)#1}, std::unique_ptr<crimson::PipelineExitBarrierI, std::default_delete<crimson::PipelineExitBarrierI> > >::run_and_dispose() at /home/sam/git-checkouts/ceph2/build/../src/seastar/include/seastar/core/future.hh:1673
 7# seastar::reactor::run_tasks(seastar::reactor::task_queue&) at /home/sam/git-checkouts/ceph2/build/../src/seastar/src/core/reactor.cc:2353
 8# seastar::reactor::run_some_tasks() at /home/sam/git-checkouts/ceph2/build/../src/seastar/src/core/reactor.cc:2766
 9# seastar::reactor::do_run() at /home/sam/git-checkouts/ceph2/build/../src/seastar/src/core/reactor.cc:2934
10# seastar::reactor::run() at /home/sam/git-checkouts/ceph2/build/../src/seastar/src/core/reactor.cc:2817
11# seastar::app_template::run_deprecated(int, char**, std::function<void ()>&&) at /home/sam/git-checkouts/ceph2/build/../src/seastar/src/core/app-template.cc:265
12# seastar::app_template::run(int, char**, std::function<seastar::future<int> ()>&&) at /home/sam/git-checkouts/ceph2/build/../src/seastar/src/core/app-template.cc:156
13# main at /home/sam/git-checkouts/ceph2/build/../src/crimson/osd/main.cc:231
14# __libc_start_main in /lib64/libc.so.6
15# _start in /home/sam/git-checkouts/ceph2/build/bin/crimson-osd