Ceph

    -3> 2012-07-02 23:19:25.997525 7f02f4aa0700  0 log [INF] : 0.6 restarting backfill on osd.1 from (0'0,0'0] MAX to 69'9651
    -2> 2012-07-02 23:19:26.002860 7f02f4aa0700  0 log [INF] : 0.10 restarting backfill on osd.0 from (69'10891,69'11891] 0//0//-1 to 79'12359
    -1> 2012-07-02 23:19:26.013881 7f02f4aa0700  0 log [INF] : 0.c restarting backfill on osd.0 from (69'20232,69'21232] 0//0//-1 to 79'21235
     0> 2012-07-02 23:19:38.124301 7f02f6aa4700 -1 osd/ReplicatedPG.cc: In function 'void ReplicatedPG::sub_op_modify_applied(ReplicatedPG::RepModify*)' thread 7f02f6aa4700 time 2012-07-02 23:19:38.091880
osd/ReplicatedPG.cc: 4293: FAILED assert(info.last_update <= active_rep_scrub->scrub_to)

 ceph version 0.48argonaut-122-g8b46201 (commit:8b462019f2d8653d0f7821150271ae067ac7e575)
 1: (ReplicatedPG::sub_op_modify_applied(ReplicatedPG::RepModify*)+0x4e4) [0x53ef14]
 2: (Finisher::finisher_thread_entry()+0x1d8) [0x71bb28]
 3: (()+0x7e9a) [0x7f030280fe9a]
 4: (clone()+0x6d) [0x7f0300dc44bd]
 NOTE: a copy of the executable, or `objdump -rdS <executable>` is needed to interpret this.

--- end dump of recent events ---
2012-07-02 23:19:38.197380 7f02f6aa4700 -1 *** Caught signal (Aborted) **
 in thread 7f02f6aa4700

 ceph version 0.48argonaut-122-g8b46201 (commit:8b462019f2d8653d0f7821150271ae067ac7e575)
 1: /tmp/cephtest/binary/usr/local/bin/ceph-osd() [0x70887a]
 2: (()+0xfcb0) [0x7f0302817cb0]
 3: (gsignal()+0x35) [0x7f0300d08445]
 4: (abort()+0x17b) [0x7f0300d0bbab]
 5: (__gnu_cxx::__verbose_terminate_handler()+0x11d) [0x7f030165669d]
 6: (()+0xb5846) [0x7f0301654846]
 7: (()+0xb5873) [0x7f0301654873]
 8: (()+0xb596e) [0x7f030165496e]
 9: (ceph::__ceph_assert_fail(char const*, char const*, int, char const*)+0x282) [0x7bff32]
 10: (ReplicatedPG::sub_op_modify_applied(ReplicatedPG::RepModify*)+0x4e4) [0x53ef14]
 11: (Finisher::finisher_thread_entry()+0x1d8) [0x71bb28]
 12: (()+0x7e9a) [0x7f030280fe9a]
 13: (clone()+0x6d) [0x7f0300dc44bd]
 NOTE: a copy of the executable, or `objdump -rdS <executable>` is needed to interpret this.

ubuntu@teuthology:/a/sage-2012-07-02_22:54:56-smoke-master-testing-basic/4993$ cat config.yaml 
kernel: &id001
  branch: testing
  kdb: true
nuke-on-error: true
overrides:
  ceph:
    branch: master
    fs: xfs
    log-whitelist:
    - slow request
roles:
- - mon.a
  - osd.0
  - osd.1
  - osd.2
- - mds.a
  - osd.3
  - osd.4
  - osd.5
- - client.0
targets:
  ubuntu@plana42.front.sepia.ceph.com: ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABAQDLK24JC5ni+UfmxncggOIxjHNsF5J/y/iureYMSpWSQLUsVIudQstka+qcojWrVfcG21W/2kokrvLHqLWyd1ar27ovQE3pMv+nG8jxL+OpEpR72QxAH5g+e7Uv3VewKVfssbyOvtOYWIVfp3z7GtpVlCO9V0DuUgbFyoiNih93kZGpIEkv2QA6j0YA65JntISwMwvXGF+VNilUOERHKXLdmLHKIbZ5aAjxGCJMs8fhsVTfPu9l1nEXklK7bgRa7395BJa48dfONdoESoZZrr4IwE3C6acc+NIw+qGwSnNweAVFvH9C+rbN6gAvW9ax0MMeK5TveDiwaTbEB6Po5UKF
  ubuntu@plana44.front.sepia.ceph.com: ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABAQDYE0eu9E8TQwtUy89Wldp54VbNBEoO9XQf77eXXzzmNwYUFRrNX0mZV/I8GqyRJuMrPG8V4aZBthBHTtnEmQ6RAS7fVdthi/hEgwnM9cAqY3KX9mR5xJnHBc/fa5KLrnSr3Wrztf42PpQNEN5Tk55K6wWUlZOTHU3vE0j3kF+YQ5FeBhQbghztHPKFR8bOmZJp9TpbXgbvEM2RWr9bYtro1KuQOgrairyVVNWdAuwZuxSQT4soyHoSkY9JmeXKsNRAOamxH9w57mDC3PXui7r6Fp8OCWSK+GmlLTtPaZtulSCcucaZtpVae7F4s9JNxaRl5RxuUtwMRfgAHGlL2BZv
  ubuntu@plana46.front.sepia.ceph.com: ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABAQCrD6s9otJ5xCNH4nyv0iJu6AoqmlNTFd8D0X9RfFBnmOMrMBWU9kwsFzPIOsuJGbSYbA8LCtjWUwaWoXmbEFtTMitxaDXp47gbVNXknHq7TGZHkWWOwKKu+tlSQBpCVzO/rzBbvJ9fcG7tewq5XcIHz0IUXsUFuEuXR1HaTUJKic2twBpaeAGNvdd6IZ9Sz9TMkfiRV/aVdcHJ/yF8bsXi3pfRPR3puMK/Nyfq5Hz/aabQo1TSyK2o0weoWV7D8vD6S8f3D7p5/5ScBhL3zUcP85SsV47W+/hTFbU8kN1Grlv2sx0fVMB/TUB/UNVdsHKGn5Nv6zb/qMqBEx9nSeZ9
tasks:
- internal.lock_machines: 3
- internal.save_config: null
- internal.check_lock: null
- internal.connect: null
- internal.check_conflict: null
- kernel: *id001
- internal.base: null
- internal.archive: null
- internal.coredump: null
- internal.syslog: null
- internal.timer: null
- chef: null
- clock: null
- ceph:
    log-whitelist:
    - wrongly marked me down or wrong addr
    - objects unfound and apparently lost
- thrashosds: null
- kclient: null
- workunit:
    clients:
      all:
      - suites/ffsb.sh