Ceph » CephFS

http://telemetry.front.sepia.ceph.com:4000/d/jByk5HaMz/crash-spec-x-ray?orgId=1&var-sig_v2=3aaddcbc011a81127704b874d64c5efef16654f3a3ed32839c1a63bcf6589bf3

Assert condition: dnl->get_inode() == in
Assert function: void Server::set_trace_dist(ceph::ref_t<MClientReply>&, CInode*, CDentry*, MDRequestRef&)

Sanitized backtrace:

    Server::set_trace_dist(boost::intrusive_ptr<MClientReply> const&, CInode*, CDentry*, boost::intrusive_ptr<MDRequestImpl>&)
    Server::reply_client_request(boost::intrusive_ptr<MDRequestImpl>&, boost::intrusive_ptr<MClientReply> const&)
    Server::respond_to_request(boost::intrusive_ptr<MDRequestImpl>&, int)
    Batch_Getattr_Lookup::_respond(int)
    BatchOp::respond(int)
    Server::respond_to_request(boost::intrusive_ptr<MDRequestImpl>&, int)
    Server::rdlock_path_pin_ref(boost::intrusive_ptr<MDRequestImpl>&, bool, bool)
    Server::handle_client_getattr(boost::intrusive_ptr<MDRequestImpl>&, bool)
    Server::dispatch_client_request(boost::intrusive_ptr<MDRequestImpl>&)
    MDCache::dispatch_request(boost::intrusive_ptr<MDRequestImpl>&)
    MDSContext::complete(int)
    MDSCacheObject::finish_waiting(unsigned long, int)
    Locker::eval_gather(SimpleLock*, bool, bool*, std::vector<MDSContext*, std::allocator<MDSContext*> >*)
    CDentry::remove_client_lease(ClientLease*, Locker*)
    Locker::handle_client_lease(boost::intrusive_ptr<MClientLease const> const&)
    Locker::dispatch(boost::intrusive_ptr<Message const> const&)
    MDSRank::handle_message(boost::intrusive_ptr<Message const> const&)
    MDSRank::_dispatch(boost::intrusive_ptr<Message const> const&, bool)
    MDSRankDispatcher::ms_dispatch(boost::intrusive_ptr<Message const> const&)
    MDSDaemon::ms_dispatch2(boost::intrusive_ptr<Message> const&)
    DispatchQueue::entry()
    DispatchQueue::DispatchThread::entry()

{
    "assert_condition": "dnl->get_inode() == in",
    "assert_file": "mds/Server.cc",
    "assert_func": "void Server::set_trace_dist(ceph::ref_t<MClientReply>&, CInode*, CDentry*, MDRequestRef&)",
    "assert_line": 2300,
    "assert_msg": "mds/Server.cc: In function 'void Server::set_trace_dist(ceph::ref_t<MClientReply>&, CInode*, CDentry*, MDRequestRef&)' thread 7f8abaa71700 time 2022-01-19T16:07:07.299361-0600\nmds/Server.cc: 2300: FAILED ceph_assert(dnl->get_inode() == in)",
    "assert_thread_name": "ms_dispatch",
    "backtrace": [
        "/lib64/libpthread.so.0(+0x12c20) [0x7f8ac33f3c20]",
        "gsignal()",
        "abort()",
        "(ceph::__ceph_assert_fail(char const*, char const*, int, char const*)+0x1a9) [0x7f8ac4405ba3]",
        "/usr/lib64/ceph/libceph-common.so.2(+0x276d6c) [0x7f8ac4405d6c]",
        "(Server::set_trace_dist(boost::intrusive_ptr<MClientReply> const&, CInode*, CDentry*, boost::intrusive_ptr<MDRequestImpl>&)+0x1089) [0x55bdb627d949]",
        "(Server::reply_client_request(boost::intrusive_ptr<MDRequestImpl>&, boost::intrusive_ptr<MClientReply> const&)+0xcda) [0x55bdb6283d8a]",
        "(Server::respond_to_request(boost::intrusive_ptr<MDRequestImpl>&, int)+0x238) [0x55bdb62840c8]",
        "(Batch_Getattr_Lookup::_respond(int)+0xaf) [0x55bdb630a8df]",
        "(BatchOp::respond(int)+0x48) [0x55bdb6594918]",
        "(Server::respond_to_request(boost::intrusive_ptr<MDRequestImpl>&, int)+0x188) [0x55bdb6284018]",
        "(Server::rdlock_path_pin_ref(boost::intrusive_ptr<MDRequestImpl>&, bool, bool)+0x6ff) [0x55bdb628512f]",
        "(Server::handle_client_getattr(boost::intrusive_ptr<MDRequestImpl>&, bool)+0xad) [0x55bdb628762d]",
        "(Server::dispatch_client_request(boost::intrusive_ptr<MDRequestImpl>&)+0x65f) [0x55bdb62bf5af]",
        "(MDCache::dispatch_request(boost::intrusive_ptr<MDRequestImpl>&)+0x33) [0x55bdb6376193]",
        "(MDSContext::complete(int)+0x56) [0x55bdb6532c06]",
        "(MDSCacheObject::finish_waiting(unsigned long, int)+0xce) [0x55bdb6554cae]",
        "(Locker::eval_gather(SimpleLock*, bool, bool*, std::vector<MDSContext*, std::allocator<MDSContext*> >*)+0x13d6) [0x55bdb6406d66]",
        "(CDentry::remove_client_lease(ClientLease*, Locker*)+0x466) [0x55bdb646bd66]",
        "(Locker::handle_client_lease(boost::intrusive_ptr<MClientLease const> const&)+0xc6a) [0x55bdb63f77aa]",
        "(Locker::dispatch(boost::intrusive_ptr<Message const> const&)+0x134) [0x55bdb6419bf4]",
        "(MDSRank::handle_message(boost::intrusive_ptr<Message const> const&)+0xbcc) [0x55bdb622fa2c]",
        "(MDSRank::_dispatch(boost::intrusive_ptr<Message const> const&, bool)+0x7bb) [0x55bdb62323cb]",
        "(MDSRankDispatcher::ms_dispatch(boost::intrusive_ptr<Message const> const&)+0x55) [0x55bdb62329c5]",
        "(MDSDaemon::ms_dispatch2(boost::intrusive_ptr<Message> const&)+0x108) [0x55bdb62225d8]",
        "(DispatchQueue::entry()+0x126a) [0x7f8ac4649aba]",
        "(DispatchQueue::DispatchThread::entry()+0x11) [0x7f8ac46fb5d1]",
        "/lib64/libpthread.so.0(+0x817a) [0x7f8ac33e917a]",
        "clone()" 
    ],
    "ceph_version": "16.2.7",
    "crash_id": "2022-01-19T22:07:07.314026Z_bfa16d64-b959-4221-9375-32dfcc93b981",
    "entity_name": "mds.980deab16bd16b05ef933948f71c9333a65f72e7",
    "os_id": "centos",
    "os_name": "CentOS Linux",
    "os_version": "8",
    "os_version_id": "8",
    "process_name": "ceph-mds",
    "stack_sig": "7278758a86083041e8b61edeaffcf9cdde1003a6f303b704eabf2b009c3814ce",
    "timestamp": "2022-01-19T22:07:07.314026Z",
    "utsname_machine": "x86_64",
    "utsname_release": "4.18.0-193.19.1.el8_2.centos.plus.x86_64",
    "utsname_sysname": "Linux",
    "utsname_version": "#1 SMP Mon Sep 14 20:42:18 UTC 2020" 
}