Bug #13111
Updated by Sage Weil over 8 years ago
<pre> -7> 2015-08-28 17:56:08.179961 7fa382c26700 5 -- op tracker -- seq: 387, time: 2015-08-28 17:56:08.179961, event: started, op: pg_notify(7.10d(5) epoch 3032) -6> 2015-08-28 17:56:08.194586 7fa36f6ed700 1 -- 192.168.17.92:6800/33615 <== osd.8 192.168.17.91:6801/27312 33 ==== MOSDPGPush(7.1f0 3032 [PushOp(eced69f0/rbd_data.85642ae8944a.000000000000085c/head//7, version: 1769'22828, data_included: [0~4194304], data_size: 4194304, omap_header_size: 0, omap_entries_size: 0, attrset_size: 2, recovery_info: ObjectRecoveryInfo(eced69f0/rbd_data.85642ae8944a.000000000000085c/head//7@1769'22828, copy_subset: [0~4194304], clone_subset: {}), after_progress: ObjectRecoveryProgress(!first, data_recovered_to:4194304, data_complete:true, omap_recovered_to:, omap_complete:true), before_progress: ObjectRecoveryProgress(first, data_recovered_to:0, data_complete:false, omap_recovered_to:, omap_complete:false)),PushOp(c84ea5f0/rbd_data.85642ae8944a.0000000000001600/head//7, version: 2725'22829, data_included: [0~4194304], data_size: 4194304, omap_header_size: 0, omap_entries_size: 0, attrset_size: 2, recovery_info: ObjectRecoveryInfo(c84ea5f0/rbd_data.85642ae8944a.0000000000001600/head//7@2725'22829, copy_subset: [0~4194304], clone_subset: {}), after_progress: ObjectRecoveryProgress(!first, data_recovered_to:4194304, data_complete:true, omap_recovered_to:, omap_complete:true), before_progress: ObjectRecoveryProgress(first, data_recovered_to:0, data_complete:false, omap_recovered_to:, omap_complete:false))]) v2 ==== 8390517+0+0 (2584546233 0 0) 0x1147b800 con 0x4615e40 -5> 2015-08-28 17:56:08.194648 7fa36f6ed700 5 -- op tracker -- seq: 388, time: 2015-08-28 17:56:08.166005, event: header_read, op: MOSDPGPush(7.1f0 3032 [PushOp(eced69f0/rbd_data.85642ae8944a.000000000000085c/head//7, version: 1769'22828, data_included: [0~4194304], data_size: 4194304, omap_header_size: 0, omap_entries_size: 0, attrset_size: 2, recovery_info: ObjectRecoveryInfo(eced69f0/rbd_data.85642ae8944a.000000000000085c/head//7@1769'22828, copy_subset: [0~4194304], clone_subset: {}), after_progress: ObjectRecoveryProgress(!first, data_recovered_to:4194304, data_complete:true, omap_recovered_to:, omap_complete:true), before_progress: ObjectRecoveryProgress(first, data_recovered_to:0, data_complete:false, omap_recovered_to:, omap_complete:false)),PushOp(c84ea5f0/rbd_data.85642ae8944a.0000000000001600/head//7, version: 2725'22829, data_included: [0~4194304], data_size: 4194304, omap_header_size: 0, omap_entries_size: 0, attrset_size: 2, recovery_info: ObjectRecoveryInfo(c84ea5f0/rbd_data.85642ae8944a.0000000000001600/head//7@2725'22829, copy_subset: [0~4194304], clone_subset: {}), after_progress: ObjectRecoveryProgress(!first, data_recovered_to:4194304, data_complete:true, omap_recovered_to:, omap_complete:true), before_progress: ObjectRecoveryProgress(first, data_recovered_to:0, data_complete:false, omap_recovered_to:, omap_complete:false))]) -4> 2015-08-28 17:56:08.194670 7fa36f6ed700 5 -- op tracker -- seq: 388, time: 2015-08-28 17:56:08.166008, event: throttled, op: MOSDPGPush(7.1f0 3032 [PushOp(eced69f0/rbd_data.85642ae8944a.000000000000085c/head//7, version: 1769'22828, data_included: [0~4194304], data_size: 4194304, omap_header_size: 0, omap_entries_size: 0, attrset_size: 2, recovery_info: ObjectRecoveryInfo(eced69f0/rbd_data.85642ae8944a.000000000000085c/head//7@1769'22828, copy_subset: [0~4194304], clone_subset: {}), after_progress: ObjectRecoveryProgress(!first, data_recovered_to:4194304, data_complete:true, omap_recovered_to:, omap_complete:true), before_progress: ObjectRecoveryProgress(first, data_recovered_to:0, data_complete:false, omap_recovered_to:, omap_complete:false)),PushOp(c84ea5f0/rbd_data.85642ae8944a.0000000000001600/head//7, version: 2725'22829, data_included: [0~4194304], data_size: 4194304, omap_header_size: 0, omap_entries_size: 0, attrset_size: 2, recovery_info: ObjectRecoveryInfo(c84ea5f0/rbd_data.85642ae8944a.0000000000001600/head//7@2725'22829, copy_subset: [0~4194304], clone_subset: {}), after_progress: ObjectRecoveryProgress(!first, data_recovered_to:4194304, data_complete:true, omap_recovered_to:, omap_complete:true), before_progress: ObjectRecoveryProgress(first, data_recovered_to:0, data_complete:false, omap_recovered_to:, omap_complete:false))]) -3> 2015-08-28 17:56:08.194688 7fa36f6ed700 5 -- op tracker -- seq: 388, time: 2015-08-28 17:56:08.194575, event: all_read, op: MOSDPGPush(7.1f0 3032 [PushOp(eced69f0/rbd_data.85642ae8944a.000000000000085c/head//7, version: 1769'22828, data_included: [0~4194304], data_size: 4194304, omap_header_size: 0, omap_entries_size: 0, attrset_size: 2, recovery_info: ObjectRecoveryInfo(eced69f0/rbd_data.85642ae8944a.000000000000085c/head//7@1769'22828, copy_subset: [0~4194304], clone_subset: {}), after_progress: ObjectRecoveryProgress(!first, data_recovered_to:4194304, data_complete:true, omap_recovered_to:, omap_complete:true), before_progress: ObjectRecoveryProgress(first, data_recovered_to:0, data_complete:false, omap_recovered_to:, omap_complete:false)),PushOp(c84ea5f0/rbd_data.85642ae8944a.0000000000001600/head//7, version: 2725'22829, data_included: [0~4194304], data_size: 4194304, omap_header_size: 0, omap_entries_size: 0, attrset_size: 2, recovery_info: ObjectRecoveryInfo(c84ea5f0/rbd_data.85642ae8944a.0000000000001600/head//7@2725'22829, copy_subset: [0~4194304], clone_subset: {}), after_progress: ObjectRecoveryProgress(!first, data_recovered_to:4194304, data_complete:true, omap_recovered_to:, omap_complete:true), before_progress: ObjectRecoveryProgress(first, data_recovered_to:0, data_complete:false, omap_recovered_to:, omap_complete:false))]) -2> 2015-08-28 17:56:08.194705 7fa36f6ed700 5 -- op tracker -- seq: 388, time: 0.000000, event: dispatched, op: MOSDPGPush(7.1f0 3032 [PushOp(eced69f0/rbd_data.85642ae8944a.000000000000085c/head//7, version: 1769'22828, data_included: [0~4194304], data_size: 4194304, omap_header_size: 0, omap_entries_size: 0, attrset_size: 2, recovery_info: ObjectRecoveryInfo(eced69f0/rbd_data.85642ae8944a.000000000000085c/head//7@1769'22828, copy_subset: [0~4194304], clone_subset: {}), after_progress: ObjectRecoveryProgress(!first, data_recovered_to:4194304, data_complete:true, omap_recovered_to:, omap_complete:true), before_progress: ObjectRecoveryProgress(first, data_recovered_to:0, data_complete:false, omap_recovered_to:, omap_complete:false)),PushOp(c84ea5f0/rbd_data.85642ae8944a.0000000000001600/head//7, version: 2725'22829, data_included: [0~4194304], data_size: 4194304, omap_header_size: 0, omap_entries_size: 0, attrset_size: 2, recovery_info: ObjectRecoveryInfo(c84ea5f0/rbd_data.85642ae8944a.0000000000001600/head//7@2725'22829, copy_subset: [0~4194304], clone_subset: {}), after_progress: ObjectRecoveryProgress(!first, data_recovered_to:4194304, data_complete:true, omap_recovered_to:, omap_complete:true), before_progress: ObjectRecoveryProgress(first, data_recovered_to:0, data_complete:false, omap_recovered_to:, omap_complete:false))]) -1> 2015-08-28 17:56:08.194765 7fa379c14700 5 -- op tracker -- seq: 388, time: 2015-08-28 17:56:08.194765, event: reached_pg, op: MOSDPGPush(7.1f0 3032 [PushOp(eced69f0/rbd_data.85642ae8944a.000000000000085c/head//7, version: 1769'22828, data_included: [0~4194304], data_size: 4194304, omap_header_size: 0, omap_entries_size: 0, attrset_size: 2, recovery_info: ObjectRecoveryInfo(eced69f0/rbd_data.85642ae8944a.000000000000085c/head//7@1769'22828, copy_subset: [0~4194304], clone_subset: {}), after_progress: ObjectRecoveryProgress(!first, data_recovered_to:4194304, data_complete:true, omap_recovered_to:, omap_complete:true), before_progress: ObjectRecoveryProgress(first, data_recovered_to:0, data_complete:false, omap_recovered_to:, omap_complete:false)),PushOp(c84ea5f0/rbd_data.85642ae8944a.0000000000001600/head//7, version: 2725'22829, data_included: [0~4194304], data_size: 4194304, omap_header_size: 0, omap_entries_size: 0, attrset_size: 2, recovery_info: ObjectRecoveryInfo(c84ea5f0/rbd_data.85642ae8944a.0000000000001600/head//7@2725'22829, copy_subset: [0~4194304], clone_subset: {}), after_progress: ObjectRecoveryProgress(!first, data_recovered_to:4194304, data_complete:true, omap_recovered_to:, omap_complete:true), before_progress: ObjectRecoveryProgress(first, data_recovered_to:0, data_complete:false, omap_recovered_to:, omap_complete:false))]) 0> 2015-08-28 17:56:08.200992 7fa379c14700 -1 osd/ReplicatedPG.cc: In function 'virtual void ReplicatedPG::on_local_recover(const hobject_t&, const object_stat_sum_t&, const ObjectRecoveryInfo&, ObjectContextRef, ObjectStore::Transaction*)' thread 7fa379c14700 time 2015-08-28 17:56:08.194843 osd/ReplicatedPG.cc: 211: FAILED assert(is_primary()) ceph version 0.94.2 (5fb85614ca8f354284c713a2f9c610860720bbf3) 1: (ceph::__ceph_assert_fail(char const*, char const*, int, char const*)+0x8b) [0xbc2b8b] 2: (ReplicatedPG::on_local_recover(hobject_t const&, object_stat_sum_t const&, ObjectRecoveryInfo const&, std::tr1::shared_ptr<ObjectContext>, ObjectStore::Transaction*)+0x4b8) [0x854ba8] 3: (ReplicatedBackend::handle_push(pg_shard_t, PushOp&, PushReplyOp*, ObjectStore::Transaction*)+0x370) [0xa0c470] 4: (ReplicatedBackend::_do_push(std::tr1::shared_ptr<OpRequest>)+0x117) [0xa0c7f7] 5: (ReplicatedBackend::handle_message(std::tr1::shared_ptr<OpRequest>)+0x40f) [0xa1767f] 6: (ReplicatedPG::do_request(std::tr1::shared_ptr<OpRequest>&, ThreadPool::TPHandle&)+0x167) [0x840f57] 7: (OSD::dequeue_op(boost::intrusive_ptr<PG>, std::tr1::shared_ptr<OpRequest>, ThreadPool::TPHandle&)+0x3d5) [0x6a0f85] 8: (OSD::ShardedOpWQ::_process(unsigned int, ceph::heartbeat_handle_d*)+0x331) [0x6a14d1] 9: (ShardedThreadPool::shardedthreadpool_worker(unsigned int)+0x875) [0xbb2c85] 10: (ShardedThreadPool::WorkThreadSharded::entry()+0x10) [0xbb4da0] 11: (()+0x8182) [0x7fa39654e182] 12: (clone()+0x6d) [0x7fa394ab900d] NOTE: a copy of the executable, or `objdump -rdS <executable>` is needed to interpret this. </pre>