Ceph

Logs: ubuntu@teuthology:/a/teuthology-2012-08-28_00:00:10-regression-next-testing-basic/10315

2012-08-28 01:18:00.303369 7f70858f9700 -1 *** Caught signal (Aborted) **
 in thread 7f70858f9700

 ceph version 0.51-272-g15995ea (commit:15995ea1c4fdd87301a36f8d4127dca6bbff1e29)
 1: /tmp/cephtest/binary/usr/local/bin/ceph-osd() [0x72ca01]
 2: (()+0xfcb0) [0x7f7096988cb0]
 3: (gsignal()+0x35) [0x7f7094c68445]
 4: (abort()+0x17b) [0x7f7094c6bbab]
 5: (__gnu_cxx::__verbose_terminate_handler()+0x11d) [0x7f70955b669d]
 6: (()+0xb5846) [0x7f70955b4846]
 7: (()+0xb5873) [0x7f70955b4873]
 8: (()+0xb596e) [0x7f70955b496e]
 9: (ceph::__ceph_assert_fail(char const*, char const*, int, char const*)+0x1ea) [0x7d5e3a]
 10: (ReplicatedPG::apply_and_flush_repops(bool)+0x743) [0x54c9a3]
 11: (ReplicatedPG::on_change()+0x1a0) [0x556800]
 12: (PG::start_peering_interval(std::tr1::shared_ptr<OSDMap const>, std::vector<int, std::allocator<int> > const&, std::vector<int, std::allocator<int> > const&)+0x5f1) [0x644f71]
 13: (PG::RecoveryState::Reset::react(PG::AdvMap const&)+0x32f) [0x64ea9f]
 14: (boost::statechart::detail::reaction_result boost::statechart::simple_state<PG::RecoveryState::Reset, PG::RecoveryState::RecoveryMachine, boost::mpl::list<mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na>, (boost::statechart::history_mode)0>::local_react_impl_non_empty::local_react_impl<boost::mpl::list5<boost::statechart::custom_reaction<PG::AdvMap>, boost::statechart::custom_reaction<PG::ActMap>, boost::statechart::custom_reaction<PG::NullEvt>, boost::statechart::custom_reaction<PG::FlushedEvt>, boost::statechart::transition<boost::statechart::event_base, PG::RecoveryState::Crashed, boost::statechart::detail::no_context<boost::statechart::event_base>, &(boost::statechart::detail::no_context<boost::statechart::event_base>::no_function(boost::statechart::event_base const&))> >, boost::statechart::simple_state<PG::RecoveryState::Reset, PG::RecoveryState::RecoveryMachine, boost::mpl::list<mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na>, (boost::statechart::history_mode)0> >(boost::statechart::simple_state<PG::RecoveryState::Reset, PG::RecoveryS
tate::RecoveryMachine, boost::mpl::list<mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na>, (boost::statechart::history_mode)0>&, boost::statechart::event_base const&, void const*)+0xf1) [0x67ed71]
 15: (boost::statechart::simple_state<PG::RecoveryState::Reset, PG::RecoveryState::RecoveryMachine, boost::mpl::list<mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na>, (boost::statechart::history_mode)0>::react_impl(boost::statechart::event_base const&, void const*)+0x4f) [0x67ee5f]
 16: (boost::statechart::state_machine<PG::RecoveryState::RecoveryMachine, PG::RecoveryState::Initial, std::allocator<void>, boost::statechart::null_exception_translator>::process_queued_events()+0xf3) [0x6733e3]
 17: (PG::RecoveryState::handle_event(boost::statechart::event_base const&, PG::RecoveryCtx*)+0x5c) [0x67366c]
 18: (PG::handle_advance_map(std::tr1::shared_ptr<OSDMap const>, std::tr1::shared_ptr<OSDMap const>, std::vector<int, std::allocator<int> >&, std::vector<int, std::allocator<int> >&, PG::RecoveryCtx*)+0x34e) [0x661f4e]
 19: (OSD::advance_pg(unsigned int, PG*, PG::RecoveryCtx*)+0x1e5) [0x5e8e15]
 20: (OSD::process_peering_events(std::list<PG*, std::allocator<PG*> > const&)+0x1c1) [0x5e91b1]
 21: (OSD::PeeringWQ::_process(std::list<PG*, std::allocator<PG*> > const&)+0x19) [0x619c49]
 22: (ThreadPool::worker()+0x523) [0x7cc193]
 23: (ThreadPool::WorkThread::entry()+0xd) [0x5fa37d]
 24: (()+0x7e9a) [0x7f7096980e9a]
 25: (clone()+0x6d) [0x7f7094d244bd]
 NOTE: a copy of the executable, or `objdump -rdS <executable>` is needed to interpret this.

--- begin dump of recent events ---
     0> 2012-08-28 01:18:00.303369 7f70858f9700 -1 *** Caught signal (Aborted) **
 in thread 7f70858f9700

 ceph version 0.51-272-g15995ea (commit:15995ea1c4fdd87301a36f8d4127dca6bbff1e29)
 1: /tmp/cephtest/binary/usr/local/bin/ceph-osd() [0x72ca01]
 2: (()+0xfcb0) [0x7f7096988cb0]
 3: (gsignal()+0x35) [0x7f7094c68445]
 4: (abort()+0x17b) [0x7f7094c6bbab]
 5: (__gnu_cxx::__verbose_terminate_handler()+0x11d) [0x7f70955b669d]
 6: (()+0xb5846) [0x7f70955b4846]
 7: (()+0xb5873) [0x7f70955b4873]
 8: (()+0xb596e) [0x7f70955b496e]
 9: (ceph::__ceph_assert_fail(char const*, char const*, int, char const*)+0x1ea) [0x7d5e3a]
 10: (ReplicatedPG::apply_and_flush_repops(bool)+0x743) [0x54c9a3]
 11: (ReplicatedPG::on_change()+0x1a0) [0x556800]
 12: (PG::start_peering_interval(std::tr1::shared_ptr<OSDMap const>, std::vector<int, std::allocator<int> > const&, std::vector<int, std::allocator<int> > const&)+0x5f1) [0x644f71]
 13: (PG::RecoveryState::Reset::react(PG::AdvMap const&)+0x32f) [0x64ea9f]
 14: (boost::statechart::detail::reaction_result boost::statechart::simple_state<PG::RecoveryState::Reset, PG::RecoveryState::RecoveryMachine, boost::mpl::list<mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na>, (boost::statechart::history_mode)0>::local_react_impl_non_empty::local_react_impl<boost::mpl::list5<boost::statechart::custom_reaction<PG::AdvMap>, boost::statechart::custom_reaction<PG::ActMap>, boost::statechart::custom_reaction<PG::NullEvt>, boost::statechart::custom_reaction<PG::FlushedEvt>, boost::statechart::transition<boost::statechart::event_base, PG::RecoveryState::Crashed, boost::statechart::detail::no_context<boost::statechart::event_base>, &(boost::statechart::detail::no_context<boost::statechart::event_base>::no_function(boost::statechart::event_base const&))> >, boost::statechart::simple_state<PG::RecoveryState::Reset, PG::RecoveryState::RecoveryMachine, boost::mpl::list<mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na>, (boost::statechart::history_mode)0> >(boost::statechart::simple_state<PG::RecoveryState::Reset, PG::RecoveryState::RecoveryMachine, boost::mpl::list<mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na>, (boost::statechart::history_mode)0>&, boost::statechart::event_base const&, void const*)+0xf1) [0x67ed71]
 15: (boost::statechart::simple_state<PG::RecoveryState::Reset, PG::RecoveryState::RecoveryMachine, boost::mpl::list<mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na>, (boost::statechart::history_mode)0>::react_impl(boost::statechart::event_base const&, void const*)+0x4f) [0x67ee5f]
 16: (boost::statechart::state_machine<PG::RecoveryState::RecoveryMachine, PG::RecoveryState::Initial, std::allocator<void>, boost::statechart::null_exception_translator>::process_queued_events()+0xf3) [0x6733e3]
 17: (PG::RecoveryState::handle_event(boost::statechart::event_base const&, PG::RecoveryCtx*)+0x5c) [0x67366c]
 18: (PG::handle_advance_map(std::tr1::shared_ptr<OSDMap const>, std::tr1::shared_ptr<OSDMap const>, std::vector<int, std::allocator<int> >&, std::vector<int, std::allocator<int> >&, PG::RecoveryCtx*)+0x34e) [0
x661f4e]
 19: (OSD::advance_pg(unsigned int, PG*, PG::RecoveryCtx*)+0x1e5) [0x5e8e15]
 20: (OSD::process_peering_events(std::list<PG*, std::allocator<PG*> > const&)+0x1c1) [0x5e91b1]
 21: (OSD::PeeringWQ::_process(std::list<PG*, std::allocator<PG*> > const&)+0x19) [0x619c49]
 22: (ThreadPool::worker()+0x523) [0x7cc193]
 23: (ThreadPool::WorkThread::entry()+0xd) [0x5fa37d]
 24: (()+0x7e9a) [0x7f7096980e9a]
 25: (clone()+0x6d) [0x7f7094d244bd]
 NOTE: a copy of the executable, or `objdump -rdS <executable>` is needed to interpret this.

ubuntu@teuthology:/a/teuthology-2012-08-28_00:00:10-regression-next-testing-basic/10315$ cat config.yaml 
kernel: &id001
  kdb: true
  sha1: 995fc068ddf675260098c60591989bf2ee184338
nuke-on-error: true
overrides:
  ceph:
    conf:
      global:
        ms inject socket failures: 5000
    fs: xfs
    log-whitelist:
    - slow request
    sha1: 15995ea1c4fdd87301a36f8d4127dca6bbff1e29
  workunit:
    sha1: 15995ea1c4fdd87301a36f8d4127dca6bbff1e29
roles:
- - mon.a
  - osd.0
  - osd.1
  - osd.2
- - mds.a
  - osd.3
  - osd.4
  - osd.5
- - client.0
targets:
  ubuntu@plana44.front.sepia.ceph.com: ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABAQDYE0eu9E8TQwtUy89Wldp54VbNBEoO9XQf77eXXzzmNwYUFRrNX0mZV/I8GqyRJuMrPG8V4aZBthBHTtnEmQ6RAS7fVdthi/hEgwnM9cAqY3KX9mR5xJnHBc/fa5KLrnSr3Wrztf42PpQNEN5Tk55K6wWUlZOTHU3vE0j3kF+YQ5FeBhQbghztHPKFR8bOmZJp9TpbXgbvEM2RWr9bYtro1KuQOgrairyVVNWdAuwZuxSQT4soyHoSkY9JmeXKsNRAOamxH9w57mDC3PXui7r6Fp8OCWSK+GmlLTtPaZtulSCcucaZtpVae7F4s9JNxaRl5RxuUtwMRfgAHGlL2BZv
  ubuntu@plana45.front.sepia.ceph.com: ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABAQDp3cwfZhOipCot6NiKX4cRMn4zx43QY0+5HdqzCQU2y7OrOJt3d0qvifnZPyeq8/d+aW2WL2OM8m4taz380JsP0SLmlpY8D0pGY/tN0pQDqIFd8EboMtKY6tR8unQrVzuczMqup/tkKSfdRp0zAeTiJ8qH7l9MaVcOw6WfRACb8f7APJE2gVRBrzPAdbqKzAphTRzZSz0cq722AX7XQDPT2dz7NoTp5Tk7xaQdDu2II+78B1H27IWdyYeonfy17yf9N+IA2Xzna/g5zu8apg7UvzyFmHunLyjr78dhPtR39201A0QJ5x5Qli9/UaB3LwiqnbCiGfx4xWFazdUFzxiD
  ubuntu@plana70.front.sepia.ceph.com: ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABAQDPImI9TdsBYWc5EAuRnrmrYFBJs+4HKRPVv/BLAhQRqpzeqXZNWsUwTCsAEqPGKDl3vt/6RuizxlSkWL1AdjomaVfVX6nULEzf0q3yEOrdSlpcPUUG8UFrFHo3IM+6AseIb9BtvV85WrSV8mYR+duhqV/UpgtFQTn5HhHmvP9Umx7cNvkmtYbM5kqPdWKIJlIMlDr/T7iGMd50ZcA1QFn2DeJyhsB1Izux793rS6r3EJfsQqaVO7W+sJ47zB0Q+wgDghX9LKxVV0B8ShU7ho7EzL97ZLSqKDyoDcqqP/N8CA59wKwmar//OuyBLAliqukTDGBTdrfQU+YVK13KCk2h
tasks:
- internal.lock_machines: 3
- internal.save_config: null
- internal.check_lock: null
- internal.connect: null
- internal.check_conflict: null
- kernel: *id001
- internal.base: null
- internal.archive: null
- internal.coredump: null
- internal.syslog: null
- internal.timer: null
- chef: null
- clock: null
- ceph:
    log-whitelist:
    - wrongly marked me down
    - objects unfound and apparently lost
- thrashosds:
    timeout: 1200
- rados:
    clients:
    - client.0
    objects: 500
    op_weights:
      delete: 10
      read: 45
      write: 45
    ops: 4000
ubuntu@teuthology:/a/teuthology-2012-08-28_00:00:10-regression-next-testing-basic/10315$ cat summary.yaml 
ceph-sha1: 15995ea1c4fdd87301a36f8d4127dca6bbff1e29
client.0-kernel-sha1: 995fc068ddf675260098c60591989bf2ee184338
description: collection:rados-thrash clusters:6-osd-3-machine.yaml fs:xfs.yaml msgr-failures:few.yaml
  thrashers:default.yaml workloads:readwrite.yaml
duration: 1260.3459980487823
failure_reason: 'Command failed with status 1: ''/tmp/cephtest/enable-coredump /tmp/cephtest/binary/usr/local/bin/ceph-coverage
  /tmp/cephtest/archive/coverage /tmp/cephtest/daemon-helper kill /tmp/cephtest/binary/usr/local/bin/ceph-osd
  -f -i 1 -c /tmp/cephtest/ceph.conf'''
flavor: basic
mds.a-kernel-sha1: 995fc068ddf675260098c60591989bf2ee184338
mon.a-kernel-sha1: 995fc068ddf675260098c60591989bf2ee184338
owner: scheduled_teuthology@teuthology
success: false

Logs: ubuntu@teuthology:/a/teuthology-2012-08-28_19:00:05-regression-master-testing-gcov/10868

Recent logs: ubuntu@teuthology:/a/teuthology-2012-08-30_00:00:06-regression-next-testing-basic/11576