Ceph » bluestore

http://telemetry.front.sepia.ceph.com:4000/d/jByk5HaMz/crash-spec-x-ray?orgId=1&var-sig_v2=2005126bb96e53ddaba3cd7452e3d88a0f069cfeb140eb21549ea9d5975a5f26

Assert condition: r == 0
Assert function: virtual void KernelDevice::aio_submit(IOContext*)

Sanitized backtrace:

    KernelDevice::aio_submit(IOContext*)
    BlueStore::_do_readv(BlueStore::Collection*, boost::intrusive_ptr<BlueStore::Onode>, interval_set<unsigned long, std::map> const&, ceph::buffer::list&, unsigned int, unsigned long)
    BlueStore::readv(boost::intrusive_ptr<ObjectStore::CollectionImpl>&, ghobject_t const&, interval_set<unsigned long, std::map>&, ceph::buffer::list&, unsigned int)
    ReplicatedBackend::build_push_op(ObjectRecoveryInfo const&, ObjectRecoveryProgress const&, ObjectRecoveryProgress*, PushOp*, object_stat_sum_t*, bool)
    ReplicatedBackend::prep_push(std::shared_ptr<ObjectContext>, hobject_t const&, pg_shard_t, eversion_t, interval_set<unsigned long, std::map>&, std::map<hobject_t, interval_set<unsigned long, std::map>, std::less<hobject_t>, std::allocator<std::pair<hobject_t const, interval_set<unsigned long, std::map> > > >&, PushOp*, bool, ObcLockManager&&)
    ReplicatedBackend::prep_push_to_replica(std::shared_ptr<ObjectContext>, hobject_t const&, pg_shard_t, PushOp*, bool)
    ReplicatedBackend::start_pushes(hobject_t const&, std::shared_ptr<ObjectContext>, ReplicatedBackend::RPGHandle*)
    ReplicatedBackend::recover_object(hobject_t const&, eversion_t, std::shared_ptr<ObjectContext>, std::shared_ptr<ObjectContext>, PGBackend::RecoveryHandle*)
    PrimaryLogPG::prep_backfill_object_push(hobject_t, eversion_t, std::shared_ptr<ObjectContext>, std::vector<pg_shard_t, std::allocator<pg_shard_t> >, PGBackend::RecoveryHandle*)
    PrimaryLogPG::recover_backfill(unsigned long, ThreadPool::TPHandle&, bool*)
    PrimaryLogPG::start_recovery_ops(unsigned long, ThreadPool::TPHandle&, unsigned long*)
    OSD::do_recovery(PG*, unsigned int, unsigned long, ThreadPool::TPHandle&)
    ceph::osd::scheduler::PGRecovery::run(OSD*, OSDShard*, boost::intrusive_ptr<PG>&, ThreadPool::TPHandle&)
    OSD::ShardedOpWQ::_process(unsigned int, ceph::heartbeat_handle_d*)
    ShardedThreadPool::shardedthreadpool_worker(unsigned int)
    ShardedThreadPool::WorkThreadSharded::entry()

{
    "assert_condition": "r == 0",
    "assert_file": "blk/kernel/KernelDevice.cc",
    "assert_func": "virtual void KernelDevice::aio_submit(IOContext*)",
    "assert_line": 844,
    "assert_msg": "blk/kernel/KernelDevice.cc: In function 'virtual void KernelDevice::aio_submit(IOContext*)' thread 7fe8b6ac1700 time 2022-07-20T20:59:35.387244-0600\nblk/kernel/KernelDevice.cc: 844: FAILED ceph_assert(r == 0)",
    "assert_thread_name": "tp_osd_tp",
    "backtrace": [
        "/lib/x86_64-linux-gnu/libpthread.so.0(+0x14140) [0x7fe8d31c5140]",
        "gsignal()",
        "abort()",
        "(ceph::__ceph_assert_fail(char const*, char const*, int, char const*)+0x17e) [0x564c2b64dabc]",
        "/usr/bin/ceph-osd(+0xc21c00) [0x564c2b64dc00]",
        "(KernelDevice::aio_submit(IOContext*)+0x28b) [0x564c2c2177eb]",
        "(BlueStore::_do_readv(BlueStore::Collection*, boost::intrusive_ptr<BlueStore::Onode>, interval_set<unsigned long, std::map> const&, ceph::buffer::v15_2_0::list&, unsigned int, unsigned long)+0xc5d) [0x564c2bcc11fd]",
        "(BlueStore::readv(boost::intrusive_ptr<ObjectStore::CollectionImpl>&, ghobject_t const&, interval_set<unsigned long, std::map>&, ceph::buffer::v15_2_0::list&, unsigned int)+0x22e) [0x564c2bcc1aae]",
        "(ReplicatedBackend::build_push_op(ObjectRecoveryInfo const&, ObjectRecoveryProgress const&, ObjectRecoveryProgress*, PushOp*, object_stat_sum_t*, bool)+0x3c7) [0x564c2bae67e7]",
        "(ReplicatedBackend::prep_push(std::shared_ptr<ObjectContext>, hobject_t const&, pg_shard_t, eversion_t, interval_set<unsigned long, std::map>&, std::map<hobject_t, interval_set<unsigned long, std::map>, std::less<hobject_t>, std::allocator<std::pair<hobject_t const, interval_set<unsigned long, std::map> > > >&, PushOp*, bool, ObcLockManager&&)+0x72c) [0x564c2bae87dc]",
        "(ReplicatedBackend::prep_push_to_replica(std::shared_ptr<ObjectContext>, hobject_t const&, pg_shard_t, PushOp*, bool)+0x1b7) [0x564c2bae8fb7]",
        "(ReplicatedBackend::start_pushes(hobject_t const&, std::shared_ptr<ObjectContext>, ReplicatedBackend::RPGHandle*)+0x30c) [0x564c2baead8c]",
        "(ReplicatedBackend::recover_object(hobject_t const&, eversion_t, std::shared_ptr<ObjectContext>, std::shared_ptr<ObjectContext>, PGBackend::RecoveryHandle*)+0x142) [0x564c2baeb2b2]",
        "(PrimaryLogPG::prep_backfill_object_push(hobject_t, eversion_t, std::shared_ptr<ObjectContext>, std::vector<pg_shard_t, std::allocator<pg_shard_t> >, PGBackend::RecoveryHandle*)+0x1b9) [0x564c2b838cf9]",
        "(PrimaryLogPG::recover_backfill(unsigned long, ThreadPool::TPHandle&, bool*)+0x41ed) [0x564c2b8a276d]",
        "(PrimaryLogPG::start_recovery_ops(unsigned long, ThreadPool::TPHandle&, unsigned long*)+0xf2a) [0x564c2b8a5e5a]",
        "(OSD::do_recovery(PG*, unsigned int, unsigned long, ThreadPool::TPHandle&)+0x295) [0x564c2b717725]",
        "(ceph::osd::scheduler::PGRecovery::run(OSD*, OSDShard*, boost::intrusive_ptr<PG>&, ThreadPool::TPHandle&)+0x19) [0x564c2b9ce8d9]",
        "(OSD::ShardedOpWQ::_process(unsigned int, ceph::heartbeat_handle_d*)+0x6a4) [0x564c2b737dc4]",
        "(ShardedThreadPool::shardedthreadpool_worker(unsigned int)+0x41a) [0x564c2be171aa]",
        "(ShardedThreadPool::WorkThreadSharded::entry()+0x10) [0x564c2be19780]",
        "/lib/x86_64-linux-gnu/libpthread.so.0(+0x8ea7) [0x7fe8d31b9ea7]",
        "clone()" 
    ],
    "ceph_version": "17.2.1",
    "crash_id": "2022-07-21T02:59:35.405581Z_b244c226-bd15-4d69-9186-9353dbcbb2ad",
    "entity_name": "osd.5f61a8dde9a7bd160ab88e5fa142b627c27c4976",
    "os_id": "11",
    "os_name": "Debian GNU/Linux 11 (bullseye)",
    "os_version": "11 (bullseye)",
    "os_version_id": "11",
    "process_name": "ceph-osd",
    "stack_sig": "5d6dcd0ee27266ea7316177f74cb359c90712492e10b58b4e8cec74460d8029d",
    "timestamp": "2022-07-21T02:59:35.405581Z",
    "utsname_machine": "x86_64",
    "utsname_release": "5.15.39-1-pve",
    "utsname_sysname": "Linux",
    "utsname_version": "#1 SMP PVE 5.15.39-1 (Wed, 22 Jun 2022 17:22:00 +0200)" 
}