Bug #2453
osd/OSD.h: 840: FAILED assert(last_scrub_pg.count(p))
0%
Description
0> 2012-05-19 17:24:09.113684 7fe66be3d700 -1 osd/OSD.h: In function 'void OSD::unreg_last_pg_scrub(pg_t, utime_t)' thread 7fe66be3d700 time 2012-05-19 17:24:09.095719 osd/OSD.h: 840: FAILED assert(last_scrub_pg.count(p)) ceph version 0.46-313-g4277d4d (commit:4277d4d3378dde4264e2b8d211371569219c6e4b) 1: (OSD::unreg_last_pg_scrub(pg_t, utime_t)+0x149) [0x641f49] 2: (PG::proc_primary_info(ObjectStore::Transaction&, pg_info_t const&)+0x5e) [0x63383e] 3: (PG::RecoveryState::ReplicaActive::react(PG::RecoveryState::MInfoRec const&)+0x4a) [0x633eda] 4: (boost::statechart::detail::reaction_result boost::statechart::simple_state<PG::RecoveryState::ReplicaActive, PG::RecoveryState::Started, boost::mpl::list<mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na>, (boost::statechart::history_mode)0>::local_react_impl_non_empty::local_react_impl<boost::mpl::list3<boost::statechart::custom_reaction<PG::RecoveryState::MQuery>, boost::statechart::custom_reaction<PG::RecoveryState::MInfoRec>, boost::statechart::custom_reaction<PG::RecoveryState::MLogRec> >, boost::statechart::simple_state<PG::RecoveryState::ReplicaActive, PG::RecoveryState::Started, boost::mpl::list<mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na>, (boost::statechart::history_mode)0> >(boost::statechart::simple_state<PG::RecoveryState::ReplicaActive, PG::RecoveryState::Started, boost::mpl::list<mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na>, (boost::statechart::history_mode)0>&, boost::statechart::event_base const&, void const*)+0x130) [0x6466a0] 5: (boost::statechart::simple_state<PG::RecoveryState::ReplicaActive, PG::RecoveryState::Started, boost::mpl::list<mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na>, (boost::statechart::history_mode)0>::react_impl(boost::statechart::event_base const&, void const*)+0x81) [0x646791] 6: (boost::statechart::state_machine<PG::RecoveryState::RecoveryMachine, PG::RecoveryState::Initial, std::allocator<void>, boost::statechart::null_exception_translator>::send_event(boost::statechart::event_base const&)+0x5b) [0x63dfcb] 7: (boost::statechart::state_machine<PG::RecoveryState::RecoveryMachine, PG::RecoveryState::Initial, std::allocator<void>, boost::statechart::null_exception_translator>::process_event(boost::statechart::event_base const&)+0x11) [0x63e0f1] 8: (PG::RecoveryState::handle_info(int, pg_info_t&, PG::RecoveryCtx*)+0x177) [0x616987] 9: (OSD::handle_pg_info(std::tr1::shared_ptr<OpRequest>)+0x665) [0x5d3d15] 10: (OSD::dispatch_op(std::tr1::shared_ptr<OpRequest>)+0x2a0) [0x5d7370] 11: (OSD::_dispatch(Message*)+0x191) [0x5dd4a1] 12: (OSD::ms_dispatch(Message*)+0x153) [0x5ddda3] 13: (SimpleMessenger::dispatch_entry()+0x863) [0x77fbc3] 14: (SimpleMessenger::DispatchThread::entry()+0xd) [0x746c5d] 15: (()+0x7efc) [0x7fe679b1fefc] 16: (clone()+0x6d) [0x7fe67815089d] NOTE: a copy of the executable, or `objdump -rdS <executable>` is needed to interpret this. ubuntu@teuthology:/a/next-2012-05-19_15:29:19/2460
Associated revisions
PG::RecoveryState::Stray::react(LogEvt&): reset last_pg_scrub
We need to reset the last_pg_scrub data in the osd since we
are replacing the info.
Probably fixes #2453
In cases like 2453, we hit the following backtrace:
0> 2012-05-19 17:24:09.113684 7fe66be3d700 -1 osd/OSD.h: In function 'void OSD::unreg_last_pg_scrub(pg_t, utime_t)' thread 7fe66be3d700 time 2012-05-19 17:24:09.095719
osd/OSD.h: 840: FAILED assert(last_scrub_pg.count(p))
ceph version 0.46-313-g4277d4d (commit:4277d4d3378dde4264e2b8d211371569219c6e4b)
1: (OSD::unreg_last_pg_scrub(pg_t, utime_t)+0x149) [0x641f49]
2: (PG::proc_primary_info(ObjectStore::Transaction&, pg_info_t const&)+0x5e) [0x63383e]
3: (PG::RecoveryState::ReplicaActive::react(PG::RecoveryState::MInfoRec const&)+0x4a) [0x633eda]
4: (boost::statechart::detail::reaction_result boost::statechart::simple_state<PG::RecoveryState::ReplicaActive, PG::RecoveryState::Started, boost::mpl::list<mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na>, (boost::statechart::history_mode)0>::local_react_impl_non_empty::local_react_impl<boost::mpl::list3<boost::statechart::custom_reaction<PG::RecoveryState::MQuery>, boost::statechart::custom_reaction<PG::RecoveryState::MInfoRec>, boost::statechart::custom_reaction<PG::RecoveryState::MLogRec> >, boost::statechart::simple_state<PG::RecoveryState::ReplicaActive, PG::RecoveryState::Started, boost::mpl::list<mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na>, (boost::statechart::history_mode)0> >(boost::statechart::simple_state<PG::RecoveryState::ReplicaActive, PG::RecoveryState::Started, boost::mpl::list<mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na>, (boost::statechart::history_mode)0>&, boost::statechart::event_base const&, void const*)+0x130) [0x6466a0]
5: (boost::statechart::simple_state<PG::RecoveryState::ReplicaActive, PG::RecoveryState::Started, boost::mpl::list<mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na>, (boost::statechart::history_mode)0>::react_impl(boost::statechart::event_base const&, void const*)+0x81) [0x646791]
6: (boost::statechart::state_machine<PG::RecoveryState::RecoveryMachine, PG::RecoveryState::Initial, std::allocator<void>, boost::statechart::null_exception_translator>::send_event(boost::statechart::event_base const&)+0x5b) [0x63dfcb]
7: (boost::statechart::state_machine<PG::RecoveryState::RecoveryMachine, PG::RecoveryState::Initial, std::allocator<void>, boost::statechart::null_exception_translator>::process_event(boost::statechart::event_base const&)+0x11) [0x63e0f1]
8: (PG::RecoveryState::handle_info(int, pg_info_t&, PG::RecoveryCtx*)+0x177) [0x616987]
9: (OSD::handle_pg_info(std::tr1::shared_ptr<OpRequest>)+0x665) [0x5d3d15]
10: (OSD::dispatch_op(std::tr1::shared_ptr<OpRequest>)+0x2a0) [0x5d7370]
11: (OSD::_dispatch(Message*)+0x191) [0x5dd4a1]
12: (OSD::ms_dispatch(Message*)+0x153) [0x5ddda3]
13: (SimpleMessenger::dispatch_entry()+0x863) [0x77fbc3]
14: (SimpleMessenger::DispatchThread::entry()+0xd) [0x746c5d]
15: (()+0x7efc) [0x7fe679b1fefc]
16: (clone()+0x6d) [0x7fe67815089d]
NOTE: a copy of the executable, or `objdump -rdS <executable>` is needed to interpret this.
Because we don't clear the scrub state before reseting info,
the last_scrub_stamp state in the info.history structure
changes without updating the osd state resulting in the
above assert failure.
Backport: stable
Signed-off-by: Samuel Just <sam.just@inktank.com>
PG::RecoveryState::Stray::react(LogEvt&): reset last_pg_scrub
We need to reset the last_pg_scrub data in the osd since we
are replacing the info.
Probably fixes #2453
In cases like 2453, we hit the following backtrace:
0> 2012-05-19 17:24:09.113684 7fe66be3d700 -1 osd/OSD.h: In function 'void OSD::unreg_last_pg_scrub(pg_t, utime_t)' thread 7fe66be3d700 time 2012-05-19 17:24:09.095719
osd/OSD.h: 840: FAILED assert(last_scrub_pg.count(p))
ceph version 0.46-313-g4277d4d (commit:4277d4d3378dde4264e2b8d211371569219c6e4b)
1: (OSD::unreg_last_pg_scrub(pg_t, utime_t)+0x149) [0x641f49]
2: (PG::proc_primary_info(ObjectStore::Transaction&, pg_info_t const&)+0x5e) [0x63383e]
3: (PG::RecoveryState::ReplicaActive::react(PG::RecoveryState::MInfoRec const&)+0x4a) [0x633eda]
4: (boost::statechart::detail::reaction_result boost::statechart::simple_state<PG::RecoveryState::ReplicaActive, PG::RecoveryState::Started, boost::mpl::list<mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na>, (boost::statechart::history_mode)0>::local_react_impl_non_empty::local_react_impl<boost::mpl::list3<boost::statechart::custom_reaction<PG::RecoveryState::MQuery>, boost::statechart::custom_reaction<PG::RecoveryState::MInfoRec>, boost::statechart::custom_reaction<PG::RecoveryState::MLogRec> >, boost::statechart::simple_state<PG::RecoveryState::ReplicaActive, PG::RecoveryState::Started, boost::mpl::list<mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na>, (boost::statechart::history_mode)0> >(boost::statechart::simple_state<PG::RecoveryState::ReplicaActive, PG::RecoveryState::Started, boost::mpl::list<mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na>, (boost::statechart::history_mode)0>&, boost::statechart::event_base const&, void const*)+0x130) [0x6466a0]
5: (boost::statechart::simple_state<PG::RecoveryState::ReplicaActive, PG::RecoveryState::Started, boost::mpl::list<mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na>, (boost::statechart::history_mode)0>::react_impl(boost::statechart::event_base const&, void const*)+0x81) [0x646791]
6: (boost::statechart::state_machine<PG::RecoveryState::RecoveryMachine, PG::RecoveryState::Initial, std::allocator<void>, boost::statechart::null_exception_translator>::send_event(boost::statechart::event_base const&)+0x5b) [0x63dfcb]
7: (boost::statechart::state_machine<PG::RecoveryState::RecoveryMachine, PG::RecoveryState::Initial, std::allocator<void>, boost::statechart::null_exception_translator>::process_event(boost::statechart::event_base const&)+0x11) [0x63e0f1]
8: (PG::RecoveryState::handle_info(int, pg_info_t&, PG::RecoveryCtx*)+0x177) [0x616987]
9: (OSD::handle_pg_info(std::tr1::shared_ptr<OpRequest>)+0x665) [0x5d3d15]
10: (OSD::dispatch_op(std::tr1::shared_ptr<OpRequest>)+0x2a0) [0x5d7370]
11: (OSD::_dispatch(Message*)+0x191) [0x5dd4a1]
12: (OSD::ms_dispatch(Message*)+0x153) [0x5ddda3]
13: (SimpleMessenger::dispatch_entry()+0x863) [0x77fbc3]
14: (SimpleMessenger::DispatchThread::entry()+0xd) [0x746c5d]
15: (()+0x7efc) [0x7fe679b1fefc]
16: (clone()+0x6d) [0x7fe67815089d]
NOTE: a copy of the executable, or `objdump -rdS <executable>` is needed to interpret this.
Because we don't clear the scrub state before reseting info,
the last_scrub_stamp state in the info.history structure
changes without updating the osd state resulting in the
above assert failure.
Backport: stable
Signed-off-by: Samuel Just <sam.just@inktank.com>
History
#1 Updated by Sage Weil over 11 years ago
- Priority changed from Urgent to High
#2 Updated by Sage Weil about 11 years ago
- Status changed from 12 to 7
possibly fixed by 0d8970fc813b33e7c6ba2484fbc43cce947d3f4d
#3 Updated by Sage Weil about 11 years ago
- Status changed from 7 to Resolved
#4 Updated by Sage Weil about 11 years ago
- Status changed from Resolved to In Progress
- Priority changed from High to Immediate
ubuntu@teuthology:/a/teuthology-2012-07-10_05:00:02-regression-stable-master-basic/8546
#5 Updated by Tamilarasi muthamizhan about 11 years ago
latest logs: ubuntu@teuthology:/a/teuthology-2012-07-12_05:00:04-regression-stable-master-basic/9637
ubuntu@teuthology:/a/teuthology-2012-07-12_05:00:04-regression-stable-master-basic/9637$ cat config.yaml
kernel: &id001
kdb: true
sha1: 26ce171915f348abd1f41da1ed139d93750d987f
nuke-on-error: true
overrides:
ceph:
conf:
client:
rbd cache: true
rbd cache max dirty: 0
fs: xfs
log-whitelist:
- slow request
sha1: 95e8d87bc3fb12580e4058401674b93e19df6e02
roles:
- - mon.a
- osd.0
- osd.1
- osd.2
- - mds.a
- osd.3
- osd.4
- osd.5
- - client.0
targets:
ubuntu@plana49.front.sepia.ceph.com: ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABAQCsodFx35LkTvKGlOrQT7Zt/AuvpUOoNz4sM8ovIxIjn/AJliZfup2KAhO/VRUpcndtJMy0eAp/v6wMy9bP0tqTNbeZ3q+zzzLuwnoCOQ0relggohTE7lzLDVLb/MHCQlxpLifwUrwQpVdiUfJ2B5mcGyMr2Lku4TcC3BrgPKuaXLkGHZl0aaShLfr6PbTvDjqI+IDT4E4iKd65KhzLuzDqAIwuZyoifiG+5KYYsoIZmCHidTZYHdf6utnBzP9jTQHroZR5kE/qbVnoL1tT9vvXWLcG7y2PB3UMmIOJmRXlys2/My5iRSC+1Wd9nBxcwE3BgWHknGQhrwkMl0uC+vH3
ubuntu@plana64.front.sepia.ceph.com: ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABAQDk4GmsUmC8svnRI6Xd+mRX2MwKb4RHECAeLfqTm2COfqfolS2wKGw3U92eJcyvpZ+2p82X7uBrimjZh5JgRtxJ1aGUG4Pi60+JBYF0WpohM/3aYISFegVNET9rcapdDaAi6fFB5vhT06Q/cYEO0tPrdqGb/O3oiDSurtqtfOzkdwSPWSTY/hSegXgOeG6EjuEfvnU4BbgXWkLlDQRXCdgQd35F0SlKJVgMo+J1MgMCEK4qnBMFN614P1gBSzZCBsSUGQdjYBOzZfCRlI2bUdPDtB0kyjp7o5Ns9gLd07TLw8h9oxvI7wxG16XnLOAIzPBNOaH4OztTMGg3wJ/1e26t
ubuntu@plana85.front.sepia.ceph.com: ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABAQDSQZpCxJOt7a1JhYaWcMhGTescgNDVQ1GyDxJa7hgPqjGtTVSUTm9ywivQS//tA08jL5DH79WRE+TFUE7iAPhexr+ROSsxCLJq/gaUFMcgqpZkErxrPHnS3Ce37LzUg4odIpo6Kze4bEp/ZawkKZcgRvFU9KEu95XjQl+Jb2qm60I7tkZzMUehnY+qDKVrTkQUo9EtBuz4NOf1GoOlauQTPSpBvyQpxYCybaSWUn2xZp0qAtpWBD6wAUe1wJScyNG+trkjHc/XWZmT8XnEdW7YINTpDXrjsqaJER5eSUO1zvRI2X8gzRIJviXcAjWiHSOPIZBjw90fW/eRW0FQjKF/
tasks:
- internal.lock_machines: 3
- internal.save_config: null
- internal.check_lock: null
- internal.connect: null
- internal.check_conflict: null
- kernel: *id001
- internal.base: null
- internal.archive: null
- internal.coredump: null
- internal.syslog: null
- internal.timer: null
- chef: null
- clock: null
- ceph:
log-whitelist:
- wrongly marked me down or wrong addr
- objects unfound and apparently lost
- thrashosds:
timeout: 1200
- rbd_fsx:
clients:
- client.0
ops: 20000
ubuntu@teuthology:/a/teuthology-2012-07-12_05:00:04-regression-stable-master-basic/9637$ cat summary.yaml
ceph-sha1: 95e8d87bc3fb12580e4058401674b93e19df6e02
description: collection:thrash clusters:6-osd-3-machine.yaml fs:xfs.yaml thrashers:default.yaml
workloads:rbd_fsx_cache_writethrough.yaml
duration: 1688.6754610538483
failure_reason: 'Command failed with status 1: ''/tmp/cephtest/enable-coredump /tmp/cephtest/binary/usr/local/bin/ceph-coverage
/tmp/cephtest/archive/coverage /tmp/cephtest/daemon-helper kill /tmp/cephtest/binary/usr/local/bin/ceph-osd
-f -i 4 -c /tmp/cephtest/ceph.conf'''
flavor: basic
owner: scheduled_teuthology@teuthology
success: false
#6 Updated by Samuel Just about 11 years ago
- Assignee set to Samuel Just
#7 Updated by Samuel Just about 11 years ago
- Backport set to stable
Probably fixed in c7fb964c077d369943bd5c066c5f99da6bd5f37c.
Backported to stable: ec5cd6def9817039704b6cc010f2797a700d8500
#8 Updated by Samuel Just about 11 years ago
- Status changed from In Progress to Resolved