Project

General

Profile

Actions
Copy link

Bug #3059

closed

osd:FAILED assert(waiting_for_ondisk.empty())

Added by Tamilarasi muthamizhan over 11 years ago. Updated over 11 years ago.

Status:
Resolved
Priority:
Normal
Assignee:
-
Category:
-
Target version:
-
% Done:

0%

Source:
Q/A
Tags:
Backport:
Regression:
Severity:
Reviewed:
Affected Versions:
ceph-qa-suite:
Pull request ID:
Crash signature (v1):
Crash signature (v2):

Description

Logs: ubuntu@teuthology: /a/teuthology-2012-08-28_00:00:10-regression-next-testing-basic/10302

--- begin dump of recent events ---2012-08-28 01:00:25.129405 7f9642c5c700 -1 osd/ReplicatedPG.cc: In fun
ction 'void ReplicatedPG::apply_and_flush_repops(bool)' thread 7f9642c5c700 time 2012-08-28 01:00:25.1236
68
osd/ReplicatedPG.cc: 5751: FAILED assert(waiting_for_ondisk.empty())

 ceph version 0.51-272-g15995ea (commit:15995ea1c4fdd87301a36f8d4127dca6bbff1e29)
 1: (ReplicatedPG::apply_and_flush_repops(bool)+0x743) [0x54c9a3]
 2: (ReplicatedPG::on_change()+0x1a0) [0x556800]
 3: (PG::start_peering_interval(std::tr1::shared_ptr<OSDMap const>, std::vector<int, std::allocator<int> 
> const&, std::vector<int, std::allocator<int> > const&)+0x5f1) [0x644f71]
 4: (PG::RecoveryState::Reset::react(PG::AdvMap const&)+0x32f) [0x64ea9f]
 5: (boost::statechart::detail::reaction_result boost::statechart::simple_state<PG::RecoveryState::Reset,
 PG::RecoveryState::RecoveryMachine, boost::mpl::list<mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, m
pl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::
na, mpl_::na, mpl_::na, mpl_::na, mpl_::na>, (boost::statechart::history_mode)0>::local_react_impl_non_em
pty::local_react_impl<boost::mpl::list5<boost::statechart::custom_reaction<PG::AdvMap>, boost::statechart
::custom_reaction<PG::ActMap>, boost::statechart::custom_reaction<PG::NullEvt>, boost::statechart::custom
_reaction<PG::FlushedEvt>, boost::statechart::transition<boost::statechart::event_base, PG::RecoveryState
::Crashed, boost::statechart::detail::no_context<boost::statechart::event_base>, &(boost::statechart::det
ail::no_context<boost::statechart::event_base>::no_function(boost::statechart::event_base const&))> >, bo
ost::statechart::simple_state<PG::RecoveryState::Reset, PG::RecoveryState::RecoveryMachine, boost::mpl::l
ist<mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, m
pl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na>, (boos
t::statechart::history_mode)0> >(boost::statechart::simple_state<PG::RecoveryState::Reset, PG::RecoverySt
ate::RecoveryMachine, boost::mpl::list<mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::
na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, m
pl_::na, mpl_::na, mpl_::na>, (boost::statechart::history_mode)0>&, boost::statechart::event_base const&,
 void const*)+0xf1) [0x67ed71]
 6: (boost::statechart::simple_state<PG::RecoveryState::Reset, PG::RecoveryState::RecoveryMachine, boost:
:mpl::list<mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na>, (boost::statechart::history_mode)0>::react_impl(boost::statechart::event_base const&, void const*)+0x4f) [0x67ee5f]
 7: (boost::statechart::state_machine<PG::RecoveryState::RecoveryMachine, PG::RecoveryState::Initial, std::allocator<void>, boost::statechart::null_exception_translator>::process_queued_events()+0xf3) [0x6733e3]
 8: (PG::RecoveryState::handle_event(boost::statechart::event_base const&, PG::RecoveryCtx*)+0x5c) [0x67366c]
 9: (PG::handle_advance_map(std::tr1::shared_ptr<OSDMap const>, std::tr1::shared_ptr<OSDMap const>, std::vector<int, std::allocator<int> >&, std::vector<int, std::allocator<int> >&, PG::RecoveryCtx*)+0x34e) [0x661f4e]
 10: (OSD::advance_pg(unsigned int, PG*, PG::RecoveryCtx*)+0x1e5) [0x5e8e15]
 11: (OSD::process_peering_events(std::list<PG*, std::allocator<PG*> > const&)+0x1c1) [0x5e91b1]
 12: (OSD::PeeringWQ::_process(std::list<PG*, std::allocator<PG*> > const&)+0x19) [0x619c49]
 13: (ThreadPool::worker()+0x523) [0x7cc193]
 14: (ThreadPool::WorkThread::entry()+0xd) [0x5fa37d]
 15: (()+0x7e9a) [0x7f9653ce3e9a]
 16: (clone()+0x6d) [0x7f96520874bd]
 NOTE: a copy of the executable, or `objdump -rdS <executable>` is needed to interpret this.

ubuntu@teuthology:/a/teuthology-2012-08-28_00:00:10-regression-next-testing-basic/10302$ cat config.yaml 
kernel: &id001
  kdb: true
  sha1: 995fc068ddf675260098c60591989bf2ee184338
nuke-on-error: true
overrides:
  ceph:
    conf:
      global:
        ms inject socket failures: 5000
    fs: btrfs
    log-whitelist:
    - slow request
    sha1: 15995ea1c4fdd87301a36f8d4127dca6bbff1e29
  workunit:
    sha1: 15995ea1c4fdd87301a36f8d4127dca6bbff1e29
roles:
- - mon.a
  - osd.0
  - osd.1
  - osd.2
- - mds.a
  - osd.3
  - osd.4
  - osd.5
- - client.0
targets:
  ubuntu@plana39.front.sepia.ceph.com: ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABAQDo+Kh24vRxeTQ6/n5PIIGuxrPHPRO/xMQlwoLHi7mR01cIXJMG5wet7mp2om3/5SZSDcLBHduDKrdWL142Sg5fC0zZPUggbxS7nz/UCjYBzMsOtHEUAU5Gs0KFopOCHXNEveK95ezsroMAD5+jS/IEpiooYCkrR3H+NSvUU0Ae352PlXqV0vamkYzyQyEMmhFE50ALhUXbKMve3d2mxJee5sqVZSBmQTbze9RKUA96t9iiwiheflXbN1i9WHlbBOIue5pZ5fM3/vqPWgaShfFpa0pT56QKJfjyFcDeCLOislo23E5qKAJOi5vn5BoYVtG3niNQpt/YbYGfDEHVeqt9
  ubuntu@plana55.front.sepia.ceph.com: ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABAQCdrzGTR0Fbl6sedYlwlX+FlmF6fuE3l/RTu2kzOkmG47rPEn5CI37Injb7Epc50RXCbUIfzmDqtEY6uZT3YssYrE4jvhQlynPndbn1KmiTbgxTyuumGXv7O4OOntezighA1W49phUNZys1DhdEEO8VSQAIdHrBgBLhY9DDgC4LAhrP4BSbDTN0rUXtYYHBj4aa3sJV0o3sKjpsyjjlieEQnto6JkjK6EGZCSuY+AyMZyLJjFTgMwJ9i4aC5eZoWZAWSDfDsxo8PtFR+kjUmz5uiheyn5lAzKBxmd4ZNojf7wOhSGia0ghbtUeQkdoRZXZhP2ourNn3uAguf1xt43kX
  ubuntu@plana62.front.sepia.ceph.com: ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABAQC+nI5/l38Kdw2W/qbEKrVMcnVdIxJG7hNnD7nnS3+Zx/uPiWrds26ZPrM5IY7D8Mf7sjBzUYbqsX9xGYMLLTQaeDwsZn/7RjjSg8zOS1aMP5F/AJzSQx4Nt37eLUsRHX3yA30/OQcl6sBgDjHyhSPcSuHWSnMmoy4pkDo3xpQMQMtxDG8gWq+to1hZwJbsiK9FdutEgPJg3inWM1WVc5L6NmRN2WQNEGT8HvtlBCWqX6/H/hLujQlbgyJAbeG4BriMV3gCIccJE833f/fN9KIzaMlD7qHTgWcaGk+LY84nUdNlTkNoX+L4m6WRY8/Pt9om2dOocsXyCwYLIS4heIDT
tasks:
- internal.lock_machines: 3
- internal.save_config: null
- internal.check_lock: null
- internal.connect: null
- internal.check_conflict: null
- kernel: *id001
- internal.base: null
- internal.archive: null
- internal.coredump: null
- internal.syslog: null
- internal.timer: null
- chef: null
- clock: null
- ceph:
    log-whitelist:
    - wrongly marked me down
    - objects unfound and apparently lost
- thrashosds:
    timeout: 1200
- radosbench:
    clients:
    - client.0
    time: 1800
ubuntu@teuthology:/a/teuthology-2012-08-28_00:00:10-regression-next-testing-basic/10302$ cat summary.yaml 
ceph-sha1: 15995ea1c4fdd87301a36f8d4127dca6bbff1e29
client.0-kernel-sha1: 995fc068ddf675260098c60591989bf2ee184338
description: collection:rados-thrash clusters:6-osd-3-machine.yaml fs:btrfs.yaml msgr-failures:few.yaml
  thrashers:default.yaml workloads:radosbench.yaml
duration: 2315.2218952178955
failure_reason: 'Command failed with status 1: ''/tmp/cephtest/enable-coredump /tmp/cephtest/binary/usr/local/bin/ceph-coverage
  /tmp/cephtest/archive/coverage /tmp/cephtest/daemon-helper kill /tmp/cephtest/binary/usr/local/bin/ceph-osd
  -f -i 1 -c /tmp/cephtest/ceph.conf'''
flavor: basic
mds.a-kernel-sha1: 995fc068ddf675260098c60591989bf2ee184338
mon.a-kernel-sha1: 995fc068ddf675260098c60591989bf2ee184338
owner: scheduled_teuthology@teuthology
success: false
Actions
Copy link

Also available in: Atom PDF