Changing get_hash_info() to return ECUtil::HashInfoRef() (NULL HashInfoRef) can cause this assert in submit_transaction().
2015-08-04 17:27:30.876250 7f86967fc700 10 osd.3 23 dequeue_op 0x7f864801d990 prio 63 cost 4096 latency 0.000269 osd_op(client.4189.0:1 obj-size-18967-0 [writefull 0~4096] 2.81384b9f ondisk+write+known_if_redirected e23) v5 pg pg[2.0s0( v 19'1 (0'0,19'1] local-les=23 n=1 ec=18 les/c 23/23 22/22/22) [3,1,2] r=0 lpr=22 crt=0'0 lcod 0'0 mlcod 0'0 active+clean]
2015-08-04 17:27:30.876321 7f86967fc700 20 osd.3 pg_epoch: 23 pg[2.0s0( v 19'1 (0'0,19'1] local-les=23 n=1 ec=18 les/c 23/23 22/22/22) [3,1,2] r=0 lpr=22 crt=0'0 lcod 0'0 mlcod 0'0 active+clean] op_has_sufficient_caps pool=2 (pool-jerasure ) owner=0 need_read_cap=0 need_write_cap=1 need_class_read_cap=0 need_class_write_cap=0 -> yes
2015-08-04 17:27:30.876333 7f86967fc700 10 osd.3 pg_epoch: 23 pg[2.0s0( v 19'1 (0'0,19'1] local-les=23 n=1 ec=18 les/c 23/23 22/22/22) [3,1,2] r=0 lpr=22 crt=0'0 lcod 0'0 mlcod 0'0 active+clean] handle_message: osd_op(client.4189.0:1 obj-size-18967-0 [writefull 0~4096] 2.81384b9f ondisk+write+known_if_redirected e23) v5
2015-08-04 17:27:30.876343 7f86967fc700 10 osd.3 pg_epoch: 23 pg[2.0s0( v 19'1 (0'0,19'1] local-les=23 n=1 ec=18 les/c 23/23 22/22/22) [3,1,2] r=0 lpr=22 crt=0'0 lcod 0'0 mlcod 0'0 active+clean] do_op osd_op(client.4189.0:1 obj-size-18967-0 [writefull 0~4096] 2.81384b9f ondisk+write+known_if_redirected e23) v5 may_write -> write-ordered flags ondisk+write+known_if_redirected
2015-08-04 17:27:30.876380 7f86967fc700 10 osd.3 pg_epoch: 23 pg[2.0s0( v 19'1 (0'0,19'1] local-les=23 n=1 ec=18 les/c 23/23 22/22/22) [3,1,2] r=0 lpr=22 crt=0'0 lcod 0'0 mlcod 0'0 active+clean] get_object_context: found obc in cache: 0x7f866c0031a0
2015-08-04 17:27:30.876422 7f86967fc700 10 osd.3 pg_epoch: 23 pg[2.0s0( v 19'1 (0'0,19'1] local-les=23 n=1 ec=18 les/c 23/23 22/22/22) [3,1,2] r=0 lpr=22 crt=0'0 lcod 0'0 mlcod 0'0 active+clean] get_object_context: 0x7f866c0031a0 2/81384b9f/obj-size-18967-0/head rwstate(none n=0 w=0) oi: 2/81384b9f/obj-size-18967-0/head(19'1 client.4164.0:1 wrlock_by=unknown.0.0:0 dirty|data_digest|omap_digest s 4096 uv 1 dd d54493fd od ffffffff) ssc: 0x7f866c004890 snapset: 0=[]:[]+head
2015-08-04 17:27:30.876438 7f86967fc700 10 osd.3 pg_epoch: 23 pg[2.0s0( v 19'1 (0'0,19'1] local-les=23 n=1 ec=18 les/c 23/23 22/22/22) [3,1,2] r=0 lpr=22 crt=0'0 lcod 0'0 mlcod 0'0 active+clean] find_object_context 2/81384b9f/obj-size-18967-0/head @head oi=2/81384b9f/obj-size-18967-0/head(19'1 client.4164.0:1 wrlock_by=unknown.0.0:0 dirty|data_digest|omap_digest s 4096 uv 1 dd d54493fd od ffffffff)
2015-08-04 17:27:30.876466 7f86967fc700 10 osd.3 pg_epoch: 23 pg[2.0s0( v 19'1 (0'0,19'1] local-les=23 n=1 ec=18 les/c 23/23 22/22/22) [3,1,2] r=0 lpr=22 crt=0'0 lcod 0'0 mlcod 0'0 active+clean] execute_ctx 0x7f866c0090d0
2015-08-04 17:27:30.876477 7f86967fc700 10 osd.3 pg_epoch: 23 pg[2.0s0( v 19'1 (0'0,19'1] local-les=23 n=1 ec=18 les/c 23/23 22/22/22) [3,1,2] r=0 lpr=22 crt=0'0 lcod 0'0 mlcod 0'0 active+clean] do_op 2/81384b9f/obj-size-18967-0/head [writefull 0~4096] ov 19'1 av 23'2 snapc 0=[] snapset 0=[]:[]+head
2015-08-04 17:27:30.876486 7f86967fc700 10 osd.3 pg_epoch: 23 pg[2.0s0( v 19'1 (0'0,19'1] local-les=23 n=1 ec=18 les/c 23/23 22/22/22) [3,1,2] r=0 lpr=22 crt=0'0 lcod 0'0 mlcod 0'0 active+clean] do_osd_op 2/81384b9f/obj-size-18967-0/head [writefull 0~4096]
2015-08-04 17:27:30.876499 7f86967fc700 10 osd.3 pg_epoch: 23 pg[2.0s0( v 19'1 (0'0,19'1] local-les=23 n=1 ec=18 les/c 23/23 22/22/22) [3,1,2] r=0 lpr=22 crt=0'0 lcod 0'0 mlcod 0'0 active+clean] do_osd_op writefull 0~4096
2015-08-04 17:27:30.876554 7f86967fc700 20 osd.3 pg_epoch: 23 pg[2.0s0( v 19'1 (0'0,19'1] local-les=23 n=1 ec=18 les/c 23/23 22/22/22) [3,1,2] r=0 lpr=22 crt=0'0 lcod 0'0 mlcod 0'0 active+clean] make_writeable 2/81384b9f/obj-size-18967-0/head snapset=0x7f866c0048d0 snapc=0=[]
2015-08-04 17:27:30.876590 7f86967fc700 20 osd.3 pg_epoch: 23 pg[2.0s0( v 19'1 (0'0,19'1] local-les=23 n=1 ec=18 les/c 23/23 22/22/22) [3,1,2] r=0 lpr=22 crt=0'0 lcod 0'0 mlcod 0'0 active+clean] make_writeable 2/81384b9f/obj-size-18967-0/head done, snapset=0=[]:[]+head
2015-08-04 17:27:30.876598 7f86967fc700 20 osd.3 pg_epoch: 23 pg[2.0s0( v 19'1 (0'0,19'1] local-les=23 n=1 ec=18 les/c 23/23 22/22/22) [3,1,2] r=0 lpr=22 crt=0'0 lcod 0'0 mlcod 0'0 active+clean] finish_ctx 2/81384b9f/obj-size-18967-0/head 0x7f866c0090d0 op modify
2015-08-04 17:27:30.876614 7f86967fc700 10 osd.3 pg_epoch: 23 pg[2.0s0( v 19'1 (0'0,19'1] local-les=23 n=1 ec=18 les/c 23/23 22/22/22) [3,1,2] r=0 lpr=22 crt=0'0 lcod 0'0 mlcod 0'0 active+clean] set mtime to 2015-08-04 17:27:30.875016
2015-08-04 17:27:30.876645 7f86967fc700 10 osd.3 pg_epoch: 23 pg[2.0s0( v 19'1 (0'0,19'1] local-les=23 n=1 ec=18 les/c 23/23 22/22/22) [3,1,2] r=0 lpr=22 crt=0'0 lcod 0'0 mlcod 0'0 active+clean] final snapset 0=[]:[]+head in 2/81384b9f/obj-size-18967-0/head
2015-08-04 17:27:30.876676 7f86967fc700 20 osd.3 pg_epoch: 23 pg[2.0s0( v 19'1 (0'0,19'1] local-les=23 n=1 ec=18 les/c 23/23 22/22/22) [3,1,2] r=0 lpr=22 crt=0'0 lcod 0'0 mlcod 0'0 active+clean] zeroing write result code 0
2015-08-04 17:27:30.876722 7f86967fc700 10 osd.3 pg_epoch: 23 pg[2.0s0( v 19'1 (0'0,19'1] local-les=23 n=1 ec=18 les/c 23/23 22/22/22) [3,1,2] r=0 lpr=22 crt=0'0 lcod 0'0 mlcod 0'0 active+clean] new_repop rep_tid 2 on osd_op(client.4189.0:1 obj-size-18967-0 [writefull 0~4096] 2.81384b9f ondisk+write+known_if_redirected e23) v5
2015-08-04 17:27:30.876738 7f86967fc700 7 osd.3 pg_epoch: 23 pg[2.0s0( v 19'1 (0'0,19'1] local-les=23 n=1 ec=18 les/c 23/23 22/22/22) [3,1,2] r=0 lpr=22 crt=0'0 lcod 0'0 mlcod 0'0 active+clean] issue_repop rep_tid 2 o 2/81384b9f/obj-size-18967-0/head
2015-08-04 17:27:30.876776 7f86967fc700 10 osd.3 pg_epoch: 23 pg[2.0s0( v 19'1 (0'0,19'1] local-les=23 n=1 ec=18 les/c 23/23 22/22/22) [3,1,2] r=0 lpr=22 crt=0'0 lcod 0'0 mlcod 0'0 active+clean] get_hash_info: Getting attr on 2/81384b9f/obj-size-18967-0/head
2015-08-04 17:27:30.876800 7f86967fc700 10 osd.3 pg_epoch: 23 pg[2.0s0( v 19'1 (0'0,19'1] local-les=23 n=1 ec=18 les/c 23/23 22/22/22) [3,1,2] r=0 lpr=22 crt=0'0 lcod 0'0 mlcod 0'0 active+clean] get_hash_info: not in cache 2/81384b9f/obj-size-18967-0/head
2015-08-04 17:27:30.876849 7f86967fc700 10 osd.3 pg_epoch: 23 pg[2.0s0( v 19'1 (0'0,19'1] local-les=23 n=1 ec=18 les/c 23/23 22/22/22) [3,1,2] r=0 lpr=22 crt=0'0 lcod 0'0 mlcod 0'0 active+clean] get_hash_info: found on disk, size 10
2015-08-04 17:27:30.876920 7f86967fc700 0 osd.3 pg_epoch: 23 pg[2.0s0( v 19'1 (0'0,19'1] local-les=23 n=1 ec=18 les/c 23/23 22/22/22) [3,1,2] r=0 lpr=22 crt=0'0 lcod 0'0 mlcod 0'0 active+clean] get_hash_info: Mismatch of total_chunk_size 2048
2015-08-04 17:27:30.876928 7f86967fc700 -1 osd.3 pg_epoch: 23 pg[2.0s0( v 19'1 (0'0,19'1] local-les=23 n=1 ec=18 les/c 23/23 22/22/22) [3,1,2] r=0 lpr=22 crt=0'0 lcod 0'0 mlcod 0'0 active+clean] submit_transaction: get_hash_info(2/81384b9f/obj-size-18967-0/head) returned a null pointer and there is no way to recover from such an error in this context
2015-08-04 17:27:30.891247 7f86967fc700 -1 osd/ECBackend.cc: In function 'virtual void ECBackend::submit_transaction(const hobject_t&, const eversion_t&, PGBackend::PGTransaction*, const eversion_t&, const eversion_t&, const std::vector<pg_log_entry_t>&, boost::optional<pg_hit_set_history_t>&, Context*, Context*, Context*, ceph_tid_t, osd_reqid_t, OpRequestRef)' thread 7f86967fc700 time 2015-08-04 17:27:30.876937
osd/ECBackend.cc: 1308: FAILED assert(0)
ceph version 9.0.2-1063-g111ceb9 (111ceb96b5933c690b31e6758182f198fb28e385)
1: (ceph::__ceph_assert_fail(char const*, char const*, int, char const*)+0x95) [0x1a05199]
2: (ECBackend::submit_transaction(hobject_t const&, eversion_t const&, PGBackend::PGTransaction*, eversion_t const&, eversion_t const&, std::vector<pg_log_entry_t, std::allocator<pg_log_entry_t> > const&, boost::optional<pg_hit_set_history_t>&, Context*, Context*, Context*, unsigned long, osd_reqid_t, std::tr1::shared_ptr<OpRequest>)+0x3fd) [0x17fe867]
3: (ReplicatedPG::issue_repop(ReplicatedPG::RepGather*)+0x820) [0x1659fc8]
4: (ReplicatedPG::execute_ctx(ReplicatedPG::OpContext*)+0x1b50) [0x162c5de]
5: (ReplicatedPG::do_op(std::tr1::shared_ptr<OpRequest>&)+0x4592) [0x1627590]
6: (ReplicatedPG::do_request(std::tr1::shared_ptr<OpRequest>&, ThreadPool::TPHandle&)+0x895) [0x1622915]
7: (OSD::dequeue_op(boost::intrusive_ptr<PG>, std::tr1::shared_ptr<OpRequest>, ThreadPool::TPHandle&)+0x47f) [0x139178d]
8: (PGQueueable::RunVis::operator()(std::tr1::shared_ptr<OpRequest>&)+0x5d) [0x133fadd]
9: (void boost::detail::variant::invoke_visitor<PGQueueable::RunVis>::internal_visit<std::tr1::shared_ptr<OpRequest> >(std::tr1::shared_ptr<OpRequest>&, int)+0x29) [0x1442505]
10: (boost::detail::variant::invoke_visitor<PGQueueable::RunVis>::result_type boost::detail::variant::visitation_impl_invoke_impl<boost::detail::variant::invoke_visitor<PGQueueable::RunVis>, void*, std::tr1::shared_ptr<OpRequest> >(int, boost::detail::variant::invoke_visitor<PGQueueable::RunVis>&, void*, std::tr1::shared_ptr<OpRequest>*, mpl_::bool_<false>)+0x40) [0x143e75d]
11: (boost::detail::variant::invoke_visitor<PGQueueable::RunVis>::result_type boost::detail::variant::visitation_impl_invoke<boost::detail::variant::invoke_visitor<PGQueueable::RunVis>, void*, std::tr1::shared_ptr<OpRequest>, boost::variant<std::tr1::shared_ptr<OpRequest>, PGSnapTrim, PGScrub, boost::detail::variant::void_, boost::detail::variant::void_, boost::detail::variant::void_, boost::detail::variant::void_, boost::detail::variant::void_, boost::detail::variant::void_, boost::detail::variant::void_, boost::detail::variant::void_, boost::detail::variant::void_, boost::detail::variant::void_, boost::detail::variant::void_, boost::detail::variant::void_, boost::detail::variant::void_, boost::detail::variant::void_, boost::detail::variant::void_, boost::detail::variant::void_, boost::detail::variant::void_>::has_fallback_type_>(int, boost::detail::variant::invoke_visitor<PGQueueable::RunVis>&, void*, std::tr1::shared_ptr<OpRequest>*, boost::variant<std::tr1::shared_ptr<OpRequest>, PGSnapTrim, PGScrub, boost::detail::variant::void_, boost::detail::variant::void_, boost::detail::variant::void_, boost::detail::variant::void_, boost::detail::variant::void_, boost::detail::variant::void_, boost::detail::variant::void_, boost::detail::variant::void_, boost::detail::variant::void_, boost::detail::variant::void_, boost::detail::variant::void_, boost::detail::variant::void_, boost::detail::variant::void_, boost::detail::variant::void_, boost::detail::variant::void_, boost::detail::variant::void_, boost::detail::variant::void_>::has_fallback_type_, int)+0x35) [0x143955b]
12: (boost::detail::variant::invoke_visitor<PGQueueable::RunVis>::result_type boost::detail::variant::visitation_impl<mpl_::int_<0>, boost::detail::variant::visitation_impl_step<boost::mpl::l_iter<boost::mpl::l_item<mpl_::long_<3l>, std::tr1::shared_ptr<OpRequest>, boost::mpl::l_item<mpl_::long_<2l>, PGSnapTrim, boost::mpl::l_item<mpl_::long_<1l>, PGScrub, boost::mpl::l_end> > > >, boost::mpl::l_iter<boost::mpl::l_end> >, boost::detail::variant::invoke_visitor<PGQueueable::RunVis>, void*, boost::variant<std::tr1::shared_ptr<OpRequest>, PGSnapTrim, PGScrub, boost::detail::variant::void_, boost::detail::variant::void_, boost::detail::variant::void_, boost::detail::variant::void_, boost::detail::variant::void_, boost::detail::variant::void_, boost::detail::variant::void_, boost::detail::variant::void_, boost::detail::variant::void_, boost::detail::variant::void_, boost::detail::variant::void_, boost::detail::variant::void_, boost::detail::variant::void_, boost::detail::variant::void_, boost::detail::variant::void_, boost::detail::variant::void_, boost::detail::variant::void_>::has_fallback_type_>(int, int, boost::detail::variant::invoke_visitor<PGQueueable::RunVis>&, void*, mpl_::bool_<false>, boost::variant<std::tr1::shared_ptr<OpRequest>, PGSnapTrim, PGScrub, boost::detail::variant::void_, boost::detail::variant::void_, boost::detail::variant::void_, boost::detail::variant::void_, boost::detail::variant::void_, boost::detail::variant::void_, boost::detail::variant::void_, boost::detail::variant::void_, boost::detail::variant::void_, boost::detail::variant::void_, boost::detail::variant::void_, boost::detail::variant::void_, boost::detail::variant::void_, boost::detail::variant::void_, boost::detail::variant::void_, boost::detail::variant::void_, boost::detail::variant::void_>::has_fallback_type_, mpl_::int_<0>*, boost::detail::variant::visitation_impl_step<boost::mpl::l_iter<boost::mpl::l_item<mpl_::long_<3l>, std::tr1::shared_ptr<OpRequest>, boost::mpl::l_item<mpl_::long_<2l>, PGSnapTrim, boost::mpl::l_item<mpl_::long_<1l>, PGScrub, boost::mpl::l_end> > > >, boost::mpl::l_iter<boost::mpl::l_end> >*)+0x79) [0x142d4c8]
13: (boost::detail::variant::invoke_visitor<PGQueueable::RunVis>::result_type boost::variant<std::tr1::shared_ptr<OpRequest>, PGSnapTrim, PGScrub, boost::detail::variant::void_, boost::detail::variant::void_, boost::detail::variant::void_, boost::detail::variant::void_, boost::detail::variant::void_, boost::detail::variant::void_, boost::detail::variant::void_, boost::detail::variant::void_, boost::detail::variant::void_, boost::detail::variant::void_, boost::detail::variant::void_, boost::detail::variant::void_, boost::detail::variant::void_, boost::detail::variant::void_, boost::detail::variant::void_, boost::detail::variant::void_, boost::detail::variant::void_>::internal_apply_visitor_impl<boost::detail::variant::invoke_visitor<PGQueueable::RunVis>, void*>(int, int, boost::detail::variant::invoke_visitor<PGQueueable::RunVis>&, void*)+0x40) [0x1417014]
14: (boost::detail::variant::invoke_visitor<PGQueueable::RunVis>::result_type boost::variant<std::tr1::shared_ptr<OpRequest>, PGSnapTrim, PGScrub, boost::detail::variant::void_, boost::detail::variant::void_, boost::detail::variant::void_, boost::detail::variant::void_, boost::detail::variant::void_, boost::detail::variant::void_, boost::detail::variant::void_, boost::detail::variant::void_, boost::detail::variant::void_, boost::detail::variant::void_, boost::detail::variant::void_, boost::detail::variant::void_, boost::detail::variant::void_, boost::detail::variant::void_, boost::detail::variant::void_, boost::detail::variant::void_, boost::detail::variant::void_>::internal_apply_visitor<boost::detail::variant::invoke_visitor<PGQueueable::RunVis> >(boost::detail::variant::invoke_visitor<PGQueueable::RunVis>&)+0x46) [0x13fce84]
15: (PGQueueable::RunVis::result_type boost::variant<std::tr1::shared_ptr<OpRequest>, PGSnapTrim, PGScrub, boost::detail::variant::void_, boost::detail::variant::void_, boost::detail::variant::void_, boost::detail::variant::void_, boost::detail::variant::void_, boost::detail::variant::void_, boost::detail::variant::void_, boost::detail::variant::void_, boost::detail::variant::void_, boost::detail::variant::void_, boost::detail::variant::void_, boost::detail::variant::void_, boost::detail::variant::void_, boost::detail::variant::void_, boost::detail::variant::void_, boost::detail::variant::void_, boost::detail::variant::void_>::apply_visitor<PGQueueable::RunVis>(PGQueueable::RunVis&)+0x36) [0x13e193e]
16: (PGQueueable::RunVis::result_type boost::apply_visitor<PGQueueable::RunVis, boost::variant<std::tr1::shared_ptr<OpRequest>, PGSnapTrim, PGScrub, boost::detail::variant::void_, boost::detail::variant::void_, boost::detail::variant::void_, boost::detail::variant::void_, boost::detail::variant::void_, boost::detail::variant::void_, boost::detail::variant::void_, boost::detail::variant::void_, boost::detail::variant::void_, boost::detail::variant::void_, boost::detail::variant::void_, boost::detail::variant::void_, boost::detail::variant::void_, boost::detail::variant::void_, boost::detail::variant::void_, boost::detail::variant::void_, boost::detail::variant::void_> >(PGQueueable::RunVis&, boost::variant<std::tr1::shared_ptr<OpRequest>, PGSnapTrim, PGScrub, boost::detail::variant::void_, boost::detail::variant::void_, boost::detail::variant::void_, boost::detail::variant::void_, boost::detail::variant::void_, boost::detail::variant::void_, boost::detail::variant::void_, boost::detail::variant::void_, boost::detail::variant::void_, boost::detail::variant::void_, boost::detail::variant::void_, boost::detail::variant::void_, boost::detail::variant::void_, boost::detail::variant::void_, boost::detail::variant::void_, boost::detail::variant::void_, boost::detail::variant::void_>&)+0x23) [0x13c176c]
17: (PGQueueable::run(OSD*, boost::intrusive_ptr<PG>&, ThreadPool::TPHandle&)+0x44) [0x13aa258]
18: (OSD::ShardedOpWQ::_process(unsigned int, ceph::heartbeat_handle_d*)+0x70d) [0x1390b2d]
19: (ShardedThreadPool::shardedthreadpool_worker(unsigned int)+0x5b8) [0x19f7732]
20: (ShardedThreadPool::WorkThreadSharded::entry()+0x25) [0x19f8fb5]
21: (Thread::entry_wrapper()+0xa8) [0x19ed082]
22: (Thread::_entry_func(void*)+0x18) [0x19ecfd0]
23: (()+0x8182) [0x7f86cdfab182]