Bug #3631: osdc/ObjectCacher.cc: 834: FAILED assert(ob->last_commit_tid < tid) during librbd_fsx - Ceph - Ceph

2012-12-27T20:59:22.673 INFO:teuthology.task.thrashosds.thrasher:Adding osd 1
2012-12-27T20:59:22.673 DEBUG:teuthology.orchestra.run:Running: 'LD_LIBRARY_PRELOAD=/tmp/cephtest/binary/usr/local/lib /tmp/cephtest/enable-coredump /tmp/cephtest/binary/usr/local/bin/ceph-coverage /tmp/cephtest/archive/coverage /tmp/cephtest/binary/usr/local/bin/ceph -k /tmp/cephtest/ceph.keyring -c /tmp/cephtest/ceph.conf --concise osd in 1'
2012-12-27T20:59:25.142 INFO:teuthology.task.rados.rados.0.out:finishing write tid 4 to plana4522453-42
2012-12-27T20:59:25.142 INFO:teuthology.task.rados.rados.0.out:finishing write tid 1 to plana4522453-42
2012-12-27T20:59:25.143 INFO:teuthology.task.rados.rados.0.err:Error: finished tid 1 when last_acked_tid was 4
2012-12-27T20:59:25.187 INFO:teuthology.task.rados.rados.0.err:./test/osd/RadosModel.h: In function 'virtual void WriteOp::_finish(TestOp::CallbackInfo*)' thread 7fc8b3fff700 time 2012-12-27 20:58:40.325822
2012-12-27T20:59:25.187 INFO:teuthology.task.rados.rados.0.err:./test/osd/RadosModel.h: 823: FAILED assert(0)
2012-12-27T20:59:25.187 INFO:teuthology.task.rados.rados.0.err: ceph version 0.55.1-362-gc0fe381 (c0fe3815567e7d89b0abc557eb72cb6a831540c8)
2012-12-27T20:59:25.188 INFO:teuthology.task.rados.rados.0.err: 1: (ceph::__ceph_assert_fail(char const*, char const*, int, char const*)+0x95) [0x7fc8c682f07d]
2012-12-27T20:59:25.188 INFO:teuthology.task.rados.rados.0.err: 2: (WriteOp::_finish(TestOp::CallbackInfo*)+0x143) [0x446adb]
2012-12-27T20:59:25.188 INFO:teuthology.task.rados.rados.0.err: 3: (TestOp::finish(TestOp::CallbackInfo*)+0x2e) [0x4665b8]
2012-12-27T20:59:25.188 INFO:teuthology.task.rados.rados.0.err: 4: (write_callback(void*, void*)+0x42) [0x466627]
2012-12-27T20:59:25.188 INFO:teuthology.task.rados.rados.0.err: 5: (librados::C_AioComplete::finish(int)+0x4d) [0x7fc8c67b9731]
2012-12-27T20:59:25.188 INFO:teuthology.task.rados.rados.0.err: 6: (Finisher::finisher_thread_entry()+0x342) [0x7fc8c682e1ee]
2012-12-27T20:59:25.188 INFO:teuthology.task.rados.rados.0.err: 7: (Finisher::FinisherThread::entry()+0x1c) [0x7fc8c67a2a74]
2012-12-27T20:59:25.188 INFO:teuthology.task.rados.rados.0.err: 8: (Thread::_entry_func(void*)+0x23) [0x7fc8c69cf91d]
2012-12-27T20:59:25.188 INFO:teuthology.task.rados.rados.0.err: 9: (()+0x7e9a) [0x7fc8c5e5be9a]
2012-12-27T20:59:25.189 INFO:teuthology.task.rados.rados.0.err: 10: (clone()+0x6d) [0x7fc8c56734bd]
2012-12-27T20:59:25.189 INFO:teuthology.task.rados.rados.0.err: NOTE: a copy of the executable, or `objdump -rdS <executable>` is needed to interpret this.

ubuntu@teuthology:/a/teuthology-2012-12-27_19:00:03-regression-next-testing-basic/28662$ cat summary.yaml 
ceph-sha1: c0fe3815567e7d89b0abc557eb72cb6a831540c8
client.0-kernel-sha1: ec18aeecd4de479601363849d489668d8f12410c
description: collection:rados-thrash clusters:6-osd-3-machine.yaml fs:xfs.yaml msgr-failures:few.yaml
  thrashers:default.yaml workloads:snaps-few-objects.yaml
duration: 1840.5642039775848
failure_reason: 'Command crashed: ''CEPH_CLIENT_ID=0 CEPH_CONF=/tmp/cephtest/ceph.conf
  LD_LIBRARY_PATH=/tmp/cephtest/binary/usr/local/lib /tmp/cephtest/enable-coredump
  /tmp/cephtest/binary/usr/local/bin/ceph-coverage /tmp/cephtest/archive/coverage
  /tmp/cephtest/binary/usr/local/bin/testrados --op read 100 --op write 100 --op delete
  50 --op snap_create 50 --op snap_remove 50 --op rollback 0 --op setattr 0 --op rmattr
  0 --op watch 0 --max-ops 4000 --objects 50 --max-in-flight 16 --size 4000000 --min-stride-size
  400000 --max-stride-size 800000 --max-seconds 0'''
flavor: basic
mds.a-kernel-sha1: ec18aeecd4de479601363849d489668d8f12410c
mon.a-kernel-sha1: ec18aeecd4de479601363849d489668d8f12410c
owner: scheduled_teuthology@teuthology
success: false
ubuntu@teuthology:/a/teuthology-2012-12-27_19:00:03-regression-next-testing-basic/28662$ cat config.yaml kernel: &id001
  kdb: true
  sha1: ec18aeecd4de479601363849d489668d8f12410c
nuke-on-error: true
overrides:
  ceph:
    conf:
      global:
        ms inject socket failures: 5000
    fs: xfs
    log-whitelist:
    - slow request
    sha1: c0fe3815567e7d89b0abc557eb72cb6a831540c8
  s3tests:
    branch: next
  workunit:
    sha1: c0fe3815567e7d89b0abc557eb72cb6a831540c8
roles:
- - mon.a
  - osd.0
  - osd.1
  - osd.2
- - mds.a
  - osd.3
  - osd.4
  - osd.5
- - client.0
targets:
  ubuntu@plana21.front.sepia.ceph.com: ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABAQCWF1ovToj1jAHkUM4IKQ3d0Bn2AzA38jBEZxCcRKBqNq//804bDbt2VOa+5UvmDdcHevcwEHvcMqnAlTh5fb0e8qCadYuJhFFF+8yfAsujTtRpvGqkP5AIe90iNspLizCeHmS7Ej8USlWmFAyUVQ6tvSkiLFJcIDf64y+mkdVdnd0WQ2ArNV9mffq/OfHhmUSVXg9+OnxW1gQMWdMcjDu99VIQUbuZASeH0wwhGvlNHBWseHHUyYuNRbdoe2KF8v41gd3D+pBPCuHcA+QqKgJkUNt4Na+dayC8M5Q44ehJ+L8067385ZN9buTnJbZRIezXY5uw3ANJxWMogNK9m39t
  ubuntu@plana44.front.sepia.ceph.com: ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABAQDYE0eu9E8TQwtUy89Wldp54VbNBEoO9XQf77eXXzzmNwYUFRrNX0mZV/I8GqyRJuMrPG8V4aZBthBHTtnEmQ6RAS7fVdthi/hEgwnM9cAqY3KX9mR5xJnHBc/fa5KLrnSr3Wrztf42PpQNEN5Tk55K6wWUlZOTHU3vE0j3kF+YQ5FeBhQbghztHPKFR8bOmZJp9TpbXgbvEM2RWr9bYtro1KuQOgrairyVVNWdAuwZuxSQT4soyHoSkY9JmeXKsNRAOamxH9w57mDC3PXui7r6Fp8OCWSK+GmlLTtPaZtulSCcucaZtpVae7F4s9JNxaRl5RxuUtwMRfgAHGlL2BZv
  ubuntu@plana45.front.sepia.ceph.com: ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABAQDp3cwfZhOipCot6NiKX4cRMn4zx43QY0+5HdqzCQU2y7OrOJt3d0qvifnZPyeq8/d+aW2WL2OM8m4taz380JsP0SLmlpY8D0pGY/tN0pQDqIFd8EboMtKY6tR8unQrVzuczMqup/tkKSfdRp0zAeTiJ8qH7l9MaVcOw6WfRACb8f7APJE2gVRBrzPAdbqKzAphTRzZSz0cq722AX7XQDPT2dz7NoTp5Tk7xaQdDu2II+78B1H27IWdyYeonfy17yf9N+IA2Xzna/g5zu8apg7UvzyFmHunLyjr78dhPtR39201A0QJ5x5Qli9/UaB3LwiqnbCiGfx4xWFazdUFzxiD
tasks:
- internal.lock_machines: 3
- internal.save_config: null
- internal.check_lock: null
- internal.connect: null
- internal.check_conflict: null
- kernel: *id001
- internal.base: null
- internal.archive: null
- internal.coredump: null
- internal.syslog: null
- internal.timer: null
- chef: null
- clock: null
- ceph:
    log-whitelist:
    - wrongly marked me down
    - objects unfound and apparently lost
- thrashosds:
    timeout: 1200
- rados:
    clients:
    - client.0
    objects: 50
    op_weights:
      delete: 50
      read: 100
      snap_create: 50
      snap_remove: 50
      snap_rollback: 50
      write: 100
    ops: 4000

Actions

Copy link

#5