Bug #7718: osd/PG.cc: 6062: FAILED assert(pg->want_acting.size()) - Ceph - Ceph

Actions

Copy link

Bug #7718

closed

osd/PG.cc: 6062: FAILED assert(pg->want_acting.size())

Added by Sage Weil about 10 years ago. Updated about 10 years ago.

Status:

Resolved

Priority:

Urgent

Assignee:

Samuel Just

Category:

OSD

Target version:

% Done:

Source:

Q/A

Tags:

Backport:

Regression:

Severity:

3 - minor

Reviewed:

Affected Versions:

ceph-qa-suite:

Pull request ID:

Crash signature (v1):

Crash signature (v2):

Description

   -10> 2014-03-14 11:30:28.092221 7f024ac87700 10 osd.0 pg_epoch: 844 pg[3.16s0( v 838'6876 (249'4289,838'6876] local-les=823 n=228 ec=8 les/c 823/798 819/822/805) [0,5,4]/[0,1,5] r=0 lpr=822 pi=797-821/5 bft=(4,2),(5,1) crt=668'6664 lcod 838'6875 mlcod 838'6875 active+remapped snaptrimq=[119~1,182~1,199~2,1a0~1,1bb~4,1c1~1]] handle_peering_event: epoch_sent: 844 epoch_requested: 844 Backfilled
    -9> 2014-03-14 11:30:28.092255 7f024ac87700  5 osd.0 pg_epoch: 844 pg[3.16s0( v 838'6876 (249'4289,838'6876] local-les=823 n=228 ec=8 les/c 823/798 819/822/805) [0,5,4]/[0,1,5] r=0 lpr=822 pi=797-821/5 bft=(4,2),(5,1) crt=668'6664 lcod 838'6875 mlcod 838'6875 active+remapped snaptrimq=[119~1,182~1,199~2,1a0~1,1bb~4,1c1~1]] exit Started/Primary/Active/Backfilling 30.819776 76 0.003101
    -8> 2014-03-14 11:30:28.092285 7f024ac87700  5 osd.0 pg_epoch: 844 pg[3.16s0( v 838'6876 (249'4289,838'6876] local-les=823 n=228 ec=8 les/c 823/798 819/822/805) [0,5,4]/[0,1,5] r=0 lpr=822 pi=797-821/5 bft=(4,2),(5,1) crt=668'6664 lcod 838'6875 mlcod 838'6875 active+remapped snaptrimq=[119~1,182~1,199~2,1a0~1,1bb~4,1c1~1]] enter Started/Primary/Active/Recovered
    -7> 2014-03-14 11:30:28.092329 7f024ac87700 10 osd.0 pg_epoch: 844 pg[3.16s0( v 838'6876 (249'4289,838'6876] local-les=823 n=228 ec=8 les/c 823/798 819/822/805) [0,5,4]/[0,1,5] r=0 lpr=822 pi=797-821/5 bft=(4,2),(5,1) crt=668'6664 lcod 838'6875 mlcod 838'6875 active+remapped snaptrimq=[119~1,182~1,199~2,1a0~1,1bb~4,1c1~1]] calc_acting osd.(0,0) 3.16s0( v 838'6876 (249'4289,838'6876] local-les=823 n=228 ec=8 les/c 823/798 819/822/805)
    -6> 2014-03-14 11:30:28.092353 7f024ac87700 10 osd.0 pg_epoch: 844 pg[3.16s0( v 838'6876 (249'4289,838'6876] local-les=823 n=228 ec=8 les/c 823/798 819/822/805) [0,5,4]/[0,1,5] r=0 lpr=822 pi=797-821/5 bft=(4,2),(5,1) crt=668'6664 lcod 838'6875 mlcod 838'6875 active+remapped snaptrimq=[119~1,182~1,199~2,1a0~1,1bb~4,1c1~1]] calc_acting osd.(1,1) 3.16s1( empty local-les=823 n=227 ec=8 les/c 823/798 819/822/805)
    -5> 2014-03-14 11:30:28.092371 7f024ac87700 10 osd.0 pg_epoch: 844 pg[3.16s0( v 838'6876 (249'4289,838'6876] local-les=823 n=228 ec=8 les/c 823/798 819/822/805) [0,5,4]/[0,1,5] r=0 lpr=822 pi=797-821/5 bft=(4,2),(5,1) crt=668'6664 lcod 838'6875 mlcod 838'6875 active+remapped snaptrimq=[119~1,182~1,199~2,1a0~1,1bb~4,1c1~1]] calc_acting osd.(2,2) 3.16s2( v 773'6829 (249'4289,773'6829] lb a4845d16/plana825316-15/1a1//3 local-les=0 n=10 ec=8 les/c 772/752 819/822/805)
    -4> 2014-03-14 11:30:28.092390 7f024ac87700 10 osd.0 pg_epoch: 844 pg[3.16s0( v 838'6876 (249'4289,838'6876] local-les=823 n=228 ec=8 les/c 823/798 819/822/805) [0,5,4]/[0,1,5] r=0 lpr=822 pi=797-821/5 bft=(4,2),(5,1) crt=668'6664 lcod 838'6875 mlcod 838'6875 active+remapped snaptrimq=[119~1,182~1,199~2,1a0~1,1bb~4,1c1~1]] calc_acting osd.(4,2) 3.16s2( empty local-les=823 n=228 ec=8 les/c 823/798 819/822/805)
    -3> 2014-03-14 11:30:28.092407 7f024ac87700 10 osd.0 pg_epoch: 844 pg[3.16s0( v 838'6876 (249'4289,838'6876] local-les=823 n=228 ec=8 les/c 823/798 819/822/805) [0,5,4]/[0,1,5] r=0 lpr=822 pi=797-821/5 bft=(4,2),(5,1) crt=668'6664 lcod 838'6875 mlcod 838'6875 active+remapped snaptrimq=[119~1,182~1,199~2,1a0~1,1bb~4,1c1~1]] calc_acting osd.(5,1) 3.16s1( empty local-les=823 n=228 ec=8 les/c 823/798 819/822/805)
    -2> 2014-03-14 11:30:28.092423 7f024ac87700 10 osd.0 pg_epoch: 844 pg[3.16s0( v 838'6876 (249'4289,838'6876] local-les=823 n=228 ec=8 les/c 823/798 819/822/805) [0,5,4]/[0,1,5] r=0 lpr=822 pi=797-821/5 bft=(4,2),(5,1) crt=668'6664 lcod 838'6875 mlcod 838'6875 active+remapped snaptrimq=[119~1,182~1,199~2,1a0~1,1bb~4,1c1~1]] calc_acting osd.(5,2) 3.16s2( empty local-les=823 n=227 ec=8 les/c 823/798 819/822/805)
    -1> 2014-03-14 11:30:28.092456 7f024ac87700 10 osd.0 pg_epoch: 844 pg[3.16s0( v 838'6876 (249'4289,838'6876] local-les=823 n=228 ec=8 les/c 823/798 819/822/805) [0,5,4]/[0,1,5] r=0 lpr=822 pi=797-821/5 bft=(4,2),(5,1) crt=668'6664 lcod 838'6875 mlcod 838'6875 active+remapped snaptrimq=[119~1,182~1,199~2,1a0~1,1bb~4,1c1~1]] For position 0:  selecting up[i]: (0,0)
For position 1:  backfilling up[i]: (5,1) and  failed to fill position ^A
For position 2:  backfilling up[i]: (4,2) and  failed to fill position ^B

     0> 2014-03-14 11:30:28.130902 7f024ac87700 -1 osd/PG.cc: In function 'PG::RecoveryState::Recovered::Recovered(boost::statechart::state<PG::RecoveryState::Recovered, PG::RecoveryState::Active>::my_context)' thread 7f024ac87700 time 2014-03-14 11:30:28.092491
osd/PG.cc: 6062: FAILED assert(pg->want_acting.size())

 ceph version 0.77-869-g87c911c (87c911cede50ffadcbb5cfd553df8d436a9975d1)
 1: (PG::RecoveryState::Recovered::Recovered(boost::statechart::state<PG::RecoveryState::Recovered, PG::RecoveryState::Active, boost::mpl::list<mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na>, (boost::statechart::history_mode)0>::my_context)+0x3ec) [0x812ecc]
 2: (boost::statechart::state<PG::RecoveryState::Recovered, PG::RecoveryState::Active, boost::mpl::list<mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na>, (boost::statechart::history_mode)0>::shallow_construct(boost::intrusive_ptr<PG::RecoveryState::Active> const&, boost::statechart::state_machine<PG::RecoveryState::RecoveryMachine, PG::RecoveryState::Initial, std::allocator<void>, boost::statechart::null_exception_translator>&)+0x5c) [0x833bfc]
 3: (boost::statechart::detail::safe_reaction_result boost::statechart::simple_state<PG::RecoveryState::Backfilling, PG::RecoveryState::Active, boost::mpl::list<mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na>, (boost::statechart::history_mode)0>::transit_impl<PG::RecoveryState::Recovered, PG::RecoveryState::RecoveryMachine, boost::statechart::detail::no_transition_function>(boost::statechart::detail::no_transition_function const&)+0x88) [0x83b5b8]
 4: (boost::statechart::simple_state<PG::RecoveryState::Backfilling, PG::RecoveryState::Active, boost::mpl::list<mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na>, (boost::statechart::history_mode)0>::react_impl(boost::statechart::event_base const&, void const*)+0xc8) [0x83b728]
 5: (boost::statechart::state_machine<PG::RecoveryState::RecoveryMachine, PG::RecoveryState::Initial, std::allocator<void>, boost::statechart::null_exception_translator>::send_event(boost::statechart::event_base const&)+0x5b) [0x82014b]
 6: (boost::statechart::state_machine<PG::RecoveryState::RecoveryMachine, PG::RecoveryState::Initial, std::allocator<void>, boost::statechart::null_exception_translator>::process_event(boost::statechart::event_base const&)+0x11) [0x8204a1]
 7: (PG::handle_peering_event(std::tr1::shared_ptr<PG::CephPeeringEvt>, PG::RecoveryCtx*)+0x303) [0x7d89a3]
 8: (OSD::process_peering_events(std::list<PG*, std::allocator<PG*> > const&, ThreadPool::TPHandle&)+0x2c6) [0x655ab6]
 9: (OSD::PeeringWQ::_process(std::list<PG*, std::allocator<PG*> > const&, ThreadPool::TPHandle&)+0x12) [0x6a38b2]
 10: (ThreadPool::worker(ThreadPool::WorkThread*)+0x4e6) [0xa50ac6]
 11: (ThreadPool::WorkThread::entry()+0x10) [0xa528d0]
 12: (()+0x7e9a) [0x7f02608e6e9a]
 13: (clone()+0x6d) [0x7f025f0abccd]
 NOTE: a copy of the executable, or `objdump -rdS <executable>` is needed to interpret this.

--- logging levels ---
   0/ 5 none

ubuntu@teuthology:/var/lib/teuthworker/archive/sage-2014-03-14_09:31:46-rados:thrash-wip-7709-testing-basic-plana/130212