@Myoungwon Oh Oh - can you take a look at
http://pulpito.front.sepia.ceph.com/rfriedma-2022-07-05_18:14:55-rados-wip-rf-unlocked-logs-distro-default-smithi/6915597/
I'm seeing something that looks disturbingly similar:
{
"crash_id": "2022-07-06T07:44:33.131281Z_a7155c84-92d1-45f0-b692-840bec5c29fa",
"timestamp": "2022-07-06T07:44:33.131281Z",
"process_name": "ceph-osd",
"entity_name": "osd.6",
"ceph_version": "17.0.0-13379-g9b018aeb",
"utsname_hostname": "smithi084",
"utsname_sysname": "Linux",
"utsname_release": "5.4.0-121-generic",
"utsname_version": "#137-Ubuntu SMP Wed Jun 15 13:33:07 UTC 2022",
"utsname_machine": "x86_64",
"os_name": "Ubuntu",
"os_id": "ubuntu",
"os_version_id": "20.04",
"os_version": "20.04.4 LTS (Focal Fossa)",
"assert_condition": "attrs || !recovery_state.get_pg_log().get_missing().is_missing(soid) || (it_objects != recovery_state.get_pg_log().get_log().objects.end() && it_objects->second->op pg_log_entry_t::LOST_REVERT)",
"assert_func": "ObjectContextRef PrimaryLogPG::get_object_context(const hobject_t&, bool, const std::map<std::__cxx11::basic_string<char>, ceph::buffer::v15_2_0::list, std::less<void> >)",
"assert_file": "/build/ceph-17.0.0-13379-g9b018aeb/src/osd/PrimaryLogPG.cc",
"assert_line": 11804,
"assert_thread_name": "tp_osd_tp",
"assert_msg": "/build/ceph-17.0.0-13379-g9b018aeb/src/osd/PrimaryLogPG.cc: In function 'ObjectContextRef PrimaryLogPG::get_object_context(const hobject_t&, bool, const std::map<std::__cxx11::basic_string<char>, ceph::buffer::v15_2_0::list, std::less<void> >)' thread 7fc7bcf69700 time 20\
22-07-06T07:44:33.111747+0000\n/build/ceph-17.0.0-13379-g9b018aeb/src/osd/PrimaryLogPG.cc: 11804: FAILED ceph_assert(attrs || !recovery_state.get_pg_log().get_missing().is_missing(soid) || (it_objects != recovery_state.get_pg_log().get_log().objects.end() && it_objects->second->op pg_log_\
entry_t::LOST_REVERT))\n",
"backtrace": [
"/lib/x86_64-linux-gnu/libpthread.so.0(+0x14420) [0x7fc7db64b420]",
"gsignal()",
"abort()",
"(ceph::__ceph_assert_fail(char const*, char const*, int, char const*)+0x1b7) [0x561566b6af2a]",
"ceph-osd(+0xc6c0d4) [0x561566b6b0d4]",
"(PrimaryLogPG::get_object_context(hobject_t const&, bool, std::map<std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> >, ceph::buffer::v15_2_0::list, std::less<void>, std::allocator<std::pair<std::__cxx11::basic_string<char, std::char_traits<char>, std::al\
locator<char> > const, ceph::buffer::v15_2_0::list> > > const*)+0x247) [0x561566d90097]",
"(PrimaryLogPG::get_adjacent_clones(std::shared_ptr<ObjectContext>, std::shared_ptr<ObjectContext>&, std::shared_ptr<ObjectContext>&)+0x1a2) [0x561566d92332]",
"(PrimaryLogPG::get_manifest_ref_count(std::shared_ptr<ObjectContext>, std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> >&, boost::intrusive_ptr<OpRequest>)+0x5cd) [0x561566db11ed]",
"(cls_get_manifest_ref_count(void*, std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> >)+0x76) [0x561566beb1f6]",
"/usr/lib/rados-classes/libcls_cas.so(+0xb069) [0x7fc7cdbef069]",
"ceph-osd(+0xe0709d) [0x561566d0609d]",
"(ClassHandler::ClassMethod::exec(void*, ceph::buffer::v15_2_0::list&, ceph::buffer::v15_2_0::list&)+0x67) [0x561566d063a7]",
"(PrimaryLogPG::do_osd_ops(PrimaryLogPG::OpContext*, std::vector<OSDOp, std::allocator<OSDOp> >&)+0xa89) [0x561566dd95b9]",
"(PrimaryLogPG::prepare_transaction(PrimaryLogPG::OpContext*)+0x9f) [0x561566dee30f]",
"(PrimaryLogPG::execute_ctx(PrimaryLogPG::OpContext*)+0x393) [0x561566df3013]",
"(PrimaryLogPG::do_op(boost::intrusive_ptr<OpRequest>&)+0x35f3) [0x561566df7f23]",
"(PrimaryLogPG::do_request(boost::intrusive_ptr<OpRequest>&, ThreadPool::TPHandle&)+0xe61) [0x561566e03161]",
"(OSD::dequeue_op(boost::intrusive_ptr<PG>, boost::intrusive_ptr<OpRequest>, ThreadPool::TPHandle&)+0x1c0) [0x561566c3c390]",
"(ceph::osd::scheduler::PGOpItem::run(OSD*, OSDShard*, boost::intrusive_ptr<PG>&, ThreadPool::TPHandle&)+0x66) [0x561566f73ed6]",
"(OSD::ShardedOpWQ::_process(unsigned int, ceph::heartbeat_handle_d*)+0x8d2) [0x561566c68a12]",
"(ShardedThreadPool::shardedthreadpool_worker(unsigned int)+0x403) [0x561567410723]",
"(ShardedThreadPool::WorkThreadSharded::entry()+0x14) [0x561567413764]",
"/lib/x86_64-linux-gnu/libpthread.so.0(+0x8609) [0x7fc7db63f609]",
"clone()"
]
}