Ceph » RADOS

     0> 2018-01-11 19:18:21.136 7f6aa2d3f700 -1 /build/ceph-13.0.0-4716-g6e36e8a/src/osd/ExtentCache.h: In function 'ExtentCache::pin_state::~pin_state()' thread 7f6aa2d3f700 time 2018-01-11 19:18:21.132859
/build/ceph-13.0.0-4716-g6e36e8a/src/osd/ExtentCache.h: 371: FAILED assert(tid == 0)

 ceph version 13.0.0-4716-g6e36e8a (6e36e8a14a882e37a781118b9d0a37a1a8817b2d) mimic (dev)
 1: (ceph::__ceph_assert_fail(char const*, char const*, int, char const*)+0xf5) [0x55b528e0f395]
 2: (()+0x8598cc) [0x55b528b8f8cc]
 3: (std::_Rb_tree<unsigned long, std::pair<unsigned long const, ECBackend::Op>, std::_Select1st<std::pair<unsigned long const, ECBackend::Op> >, std::less<unsigned long>, std::allocator<std::pair<unsigned long const, ECBackend::Op> > >::erase(unsigned long const&)+0x122) [0x55b528b8fdf2]
 4: (ECBackend::try_reads_to_commit()+0xb0) [0x55b528b71d50]
 5: (ECBackend::check_ops()+0x1c) [0x55b528b768ec]
 6: (ECBackend::handle_sub_write_reply(pg_shard_t, ECSubWriteReply const&, ZTracer::Trace const&)+0x136) [0x55b528b76a36]
 7: (ECBackend::sub_write_applied(unsigned long, eversion_t, ZTracer::Trace const&)+0x116) [0x55b528b770e6]
 8: (Context::complete(int)+0x9) [0x55b5288a6cb9]
 9: (PrimaryLogPG::BlessedContext::sync_finish(int)+0x19) [0x55b528a1d039]
 10: (Context::sync_complete(int)+0xe) [0x55b5288a6cde]
 11: (BlueStore::queue_transactions(ObjectStore::Sequencer*, std::vector<ObjectStore::Transaction, std::allocator<ObjectStore::Transaction> >&, boost::intrusive_ptr<TrackedOp>, ThreadPool::TPHandle*)+0xcfd) [0x55b528cd003d]
 12: (non-virtual thunk to PrimaryLogPG::queue_transactions(std::vector<ObjectStore::Transaction, std::allocator<ObjectStore::Transaction> >&, boost::intrusive_ptr<OpRequest>)+0x68) [0x55b528a247d8]
 13: (ECBackend::handle_sub_write(pg_shard_t, boost::intrusive_ptr<OpRequest>, ECSubWrite&, ZTracer::Trace const&, Context*)+0x8c8) [0x55b528b68878]
 14: (ECBackend::try_reads_to_commit()+0x1c5b) [0x55b528b738fb]
 15: (ECBackend::check_ops()+0x1c) [0x55b528b768ec]
 16: (ECBackend::handle_sub_write_reply(pg_shard_t, ECSubWriteReply const&, ZTracer::Trace const&)+0x136) [0x55b528b76a36]
 17: (ECBackend::sub_write_applied(unsigned long, eversion_t, ZTracer::Trace const&)+0x116) [0x55b528b770e6]
 18: (Context::complete(int)+0x9) [0x55b5288a6cb9]
 19: (PrimaryLogPG::BlessedContext::sync_finish(int)+0x19) [0x55b528a1d039]
 20: (Context::sync_complete(int)+0xe) [0x55b5288a6cde]
 21: (BlueStore::queue_transactions(ObjectStore::Sequencer*, std::vector<ObjectStore::Transaction, std::allocator<ObjectStore::Transaction> >&, boost::intrusive_ptr<TrackedOp>, ThreadPool::TPHandle*)+0xcfd) [0x55b528cd003d]
 22: (non-virtual thunk to PrimaryLogPG::queue_transactions(std::vector<ObjectStore::Transaction, std::allocator<ObjectStore::Transaction> >&, boost::intrusive_ptr<OpRequest>)+0x68) [0x55b528a247d8]
 23: (ECBackend::handle_sub_write(pg_shard_t, boost::intrusive_ptr<OpRequest>, ECSubWrite&, ZTracer::Trace const&, Context*)+0x8c8) [0x55b528b68878]
 24: (ECBackend::try_reads_to_commit()+0x1c5b) [0x55b528b738fb]
 25: (ECBackend::check_ops()+0x1c) [0x55b528b768ec]
 26: (CallClientContexts::finish(std::pair<RecoveryMessages*, ECBackend::read_result_t&>&)+0xc38) [0x55b528b91db8]
 27: (ECBackend::complete_read_op(ECBackend::ReadOp&, RecoveryMessages*)+0x86) [0x55b528b66676]
 28: (ECBackend::handle_sub_read_reply(pg_shard_t, ECSubReadReply&, RecoveryMessages*, ZTracer::Trace const&)+0x1088) [0x55b528b6eb08]
 29: (ECBackend::_handle_message(boost::intrusive_ptr<OpRequest>)+0x4ca) [0x55b528b7ceaa]
 30: (PGBackend::handle_message(boost::intrusive_ptr<OpRequest>)+0x97) [0x55b528a64037]
 31: (PrimaryLogPG::do_request(boost::intrusive_ptr<OpRequest>&, ThreadPool::TPHandle&)+0x675) [0x55b528a1c0b5]
 32: (OSD::dequeue_op(boost::intrusive_ptr<PG>, boost::intrusive_ptr<OpRequest>, ThreadPool::TPHandle&)+0x341) [0x55b528872931]
 33: (PGOpItem::run(OSD*, boost::intrusive_ptr<PG>&, ThreadPool::TPHandle&)+0x62) [0x55b528ae6182]
 34: (OSD::ShardedOpWQ::_process(unsigned int, ceph::heartbeat_handle_d*)+0xdfd) [0x55b528879fbd]
 35: (ShardedThreadPool::shardedthreadpool_worker(unsigned int)+0x4f2) [0x55b528e13342]
 36: (ShardedThreadPool::WorkThreadSharded::entry()+0x10) [0x55b528e15690]
 37: (()+0x76ba) [0x7f6ac66946ba]
 38: (clone()+0x6d) [0x7f6ac5ca33dd]
 NOTE: a copy of the executable, or `objdump -rdS <executable>` is needed to interpret this.

/a/sage-2018-01-11_15:01:24-rados-wip-sage3-testing-2018-01-11-0711-distro-basic-smithi/2059620

This is probably fallout from the synchronous onreadable completions?