Ceph » rbd

log: ubuntu@teuthology:/a.old/teuthology-2013-01-02_19:00:03-regression-next-testing-basic/33734

2013-01-03T02:28:45.025 INFO:teuthology.task.workunit.client.0.err:+ rbd rm test
2013-01-03T02:28:45.293 INFO:teuthology.task.workunit.client.0.err:2013-01-03 02:27:53.827814 7fb7ceef5700  0 -- 10.214.131.29:0/1015454 >> 10.214.132.30:6800/27929 pipe(0x1001000 sd=12 :47953 s=2 pgs=472 
cs=1 l=1).injecting socket failure
2013-01-03T02:28:45.335 INFO:teuthology.task.workunit.client.0.out:^MRemoving image: 4% complete...^MRemoving image: 8% complete...^MRemoving image: 12% complete...^MRemoving image: 16% complete...^MRemovi
ng image: 20% complete...^MRemoving image: 24% complete...^MRemoving image: 28% complete...^MRemoving image: 32% complete...^MRemoving image: 36% complete...^MRemoving image: 40% complete...^MRemoving imag
e: 44% complete...^MRemoving image: 48% complete...^MRemoving image: 52% complete...^MRemoving image: 56% complete...^MRemoving image: 60% complete...^MRemoving image: 64% complete...^MRemoving image: 68% 
complete...^MRemoving image: 72% complete...^MRemoving image: 76% complete...^MRemoving image: 80% complete...^MRemoving image: 84% complete...^MRemoving image: 88% complete...^MRemoving image: 92% complet
e...^MRemoving image: 96% complete...^MRemoving image: 96% complete...failed.
2013-01-03T02:28:45.335 INFO:teuthology.task.workunit.client.0.err:rbd: error: image still has watchers
2013-01-03T02:28:45.335 INFO:teuthology.task.workunit.client.0.err:This means the image is still open or the client using it crashed. Try again after closing/unmapping it or waiting 30s for the crashed cli
ent to timeout.
2013-01-03T02:28:45.336 INFO:teuthology.task.workunit.client.0.err:2013-01-03 02:27:53.869258 7fb7e3cbc780 -1 librbd: error removing header: (16) Device or resource busy
2013-01-03T02:28:49.483 INFO:teuthology.task.workunit.client.0.out:^MResizing image: 100% complete...done.
2013-01-03T02:28:49.500 INFO:teuthology.task.workunit:Stopping rbd/watch_correct_version.sh on client.0...
2013-01-03T02:28:49.500 DEBUG:teuthology.orchestra.run:Running: 'rm -rf -- /tmp/cephtest/workunits.list /tmp/cephtest/workunit.client.0'
2013-01-03T02:28:49.541 ERROR:teuthology.run_tasks:Saw exception from tasks
Traceback (most recent call last):
  File "/var/lib/teuthworker/teuthology-next/teuthology/run_tasks.py", line 25, in run_tasks
    manager = _run_one_task(taskname, ctx=ctx, config=config)
  File "/var/lib/teuthworker/teuthology-next/teuthology/run_tasks.py", line 14, in _run_one_task
    return fn(**kwargs)
  File "/var/lib/teuthworker/teuthology-next/teuthology/task/workunit.py", line 86, in task
    all_spec = True
  File "/var/lib/teuthworker/teuthology-next/teuthology/parallel.py", line 83, in __exit__
    for result in self:
  File "/var/lib/teuthworker/teuthology-next/teuthology/parallel.py", line 100, in next
    resurrect_traceback(result)
  File "/var/lib/teuthworker/teuthology-next/teuthology/parallel.py", line 19, in capture_traceback
    return func(*args, **kwargs)
  File "/var/lib/teuthworker/teuthology-next/teuthology/task/workunit.py", line 222, in _run_tests
    args=args,
  File "/var/lib/teuthworker/teuthology-next/teuthology/orchestra/remote.py", line 40, in run
    r = self._runner(client=self.ssh, **kwargs)
  File "/var/lib/teuthworker/teuthology-next/teuthology/orchestra/run.py", line 257, in run
    r.exitstatus = _check_status(r.exitstatus)
  File "/var/lib/teuthworker/teuthology-next/teuthology/orchestra/run.py", line 253, in _check_status
    raise CommandFailedError(command=r.command, exitstatus=status)
CommandFailedError: Command failed with status 16: 'mkdir -p -- /tmp/cephtest/mnt.0/client.0/tmp && cd -- /tmp/cephtest/mnt.0/client.0/tmp && CEPH_REF=43cba617aa0247d714632bddf31b9271ef3a1b50 PATH="$PATH:/
tmp/cephtest/binary/usr/local/bin" LD_LIBRARY_PATH="$LD_LIBRARY_PATH:/tmp/cephtest/binary/usr/local/lib" CEPH_CONF="/tmp/cephtest/ceph.conf" CEPH_SECRET_FILE="/tmp/cephtest/data/client.0.secret" CEPH_ID="0
" PYTHONPATH="$PYTHONPATH:/tmp/cephtest/binary/usr/local/lib/python2.7/dist-packages:/tmp/cephtest/binary/usr/local/lib/python2.6/dist-packages" /tmp/cephtest/enable-coredump /tmp/cephtest/binary/usr/local
/bin/ceph-coverage /tmp/cephtest/archive/coverage /tmp/cephtest/workunit.client.0/rbd/watch_correct_version.sh'

ubuntu@teuthology:/a.old/teuthology-2013-01-02_19:00:03-regression-next-testing-basic/33734$ cat config.yaml 
kernel: &id001
  kdb: true
  sha1: 694eba0efb52f4954e2a954cd2ed967167056245
nuke-on-error: true
overrides:
  ceph:
    conf:
      global:
        ms inject socket failures: 500
    fs: btrfs
    log-whitelist:
    - slow request
    sha1: 43cba617aa0247d714632bddf31b9271ef3a1b50
  s3tests:
    branch: next
  workunit:
    sha1: 43cba617aa0247d714632bddf31b9271ef3a1b50
roles:
- - mon.a
  - mon.c
  - osd.0
  - osd.1
  - osd.2
- - mon.b
  - mds.a
  - osd.3
  - osd.4
  - osd.5
- - client.0
targets:
  ubuntu@plana11.front.sepia.ceph.com: ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABAQDQ6pa08z6rzzYAg8XwQMQNufxFrnjLBD81wtwoHqRQucPoBh9qGWWoS0cEIiyLMp7rbSuof5XrqkGxHFP99DgengGwpuAanXA1DfmFf87mL4q8MOyD6YNzgDW9vVqq/ENHk+GlDvT823WYWwod4/p1BZA+kehQYkVF9ebpXZJBbr3Cc+4F2LK+fbxrn+1U1OC6F87j8XulcPe6csQfVljndtsEgA4KCtpxNn54fVON6559jbdgTKj8yDZjUF76m8jU8ImVyqulQ6AL6gwOONpqf1lpxRSRDm8zd7Uuo/zm25ku1MZ9B8gkL+yBZDpWZbLLEXDca/LKfrneUO0LqjzH
  ubuntu@plana48.front.sepia.ceph.com: ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABAQC1ScpN+amKWlGYSJOt8sqzLTCXYaU5QTqB3bbss463/VRcDQwabXs6tib54CO6FzWaloSqMaPZchA4HeNXOwdQ9daBnG1b0vrqs0B5jZnCVzvK9AoWGDg0tm68PWYr1AtJcNyIsutywVRdzjA2nzSioKU59dKugVog/+pkoB0hGYvXo72pePMV00IrgMr9FSbDnxi3L9iJvi2LD0Pnecx6DMnaDob/T+X5y0piap5esjwIIq7wqvXuEJE9jdmxPfHo3ise2j9UA2SGI5b7HL3YOemo0zic7ukCMvlc8Ag5dQnjANTcj2eUJyagvzJgxGqhoxyAp/WpmaHZkvf0RasB
  ubuntu@plana51.front.sepia.ceph.com: ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABAQDLTsxC+nR+xtTXMbOtazCh7MOzgBKjX/oCLMP16k0AtH8Ui92tlqsfNxcHczUol0DNzxCITgrhF8FTvgM3EgkbOUVAxGj+xLqfxsdlf58nTVXbm/pOGYnvOI8CvA4DgISHDbkzuFH4FKtR8qNTTFVmtEXaZ+jpSvn7vrYuI/Uu9XZOQh73phYW8zvVB1x8770czM0Gy2wgxdNguKy6L/Q9ShsLcFfm8Uvxf6aXb3qmuxwGhqYsMlNl0X3AjoOwmow74rodlcMvQP/pAQdjMZfe1lBPqsjmU518BE5eo7zV3O9iF6ahOrm8igOu9bfki0G52R22pA3hE9BPKPfzA0hL
tasks:
- internal.lock_machines: 3
- internal.save_config: null
- internal.check_lock: null
- internal.connect: null
- internal.check_conflict: null
- kernel: *id001
- internal.base: null
- internal.archive: null
- internal.coredump: null
- internal.syslog: null
- internal.timer: null
- chef: null
- clock: null
- ceph: null
- ceph-fuse: null
- workunit:
    clients:
      client.0:
      - rbd/watch_correct_version.sh
ubuntu@teuthology:/a.old/teuthology-2013-01-02_19:00:03-regression-next-testing-basic/33734$ cat summary.yaml 
ceph-sha1: 43cba617aa0247d714632bddf31b9271ef3a1b50
client.0-kernel-sha1: 694eba0efb52f4954e2a954cd2ed967167056245
description: collection:rbd-basic clusters:fixed-3.yaml fs:btrfs.yaml msgr-failures:many.yaml
  tasks:rbd_header_race.yaml
duration: 655.1072459220886
failure_reason: 'Command failed with status 16: ''mkdir -p -- /tmp/cephtest/mnt.0/client.0/tmp
  && cd -- /tmp/cephtest/mnt.0/client.0/tmp && CEPH_REF=43cba617aa0247d714632bddf31b9271ef3a1b50
  PATH="$PATH:/tmp/cephtest/binary/usr/local/bin" LD_LIBRARY_PATH="$LD_LIBRARY_PATH:/tmp/cephtest/binary/usr/local/lib" 
  CEPH_CONF="/tmp/cephtest/ceph.conf" CEPH_SECRET_FILE="/tmp/cephtest/data/client.0.secret" 
  CEPH_ID="0" PYTHONPATH="$PYTHONPATH:/tmp/cephtest/binary/usr/local/lib/python2.7/dist-packages:/tmp/cephtest/binary/usr/local/lib/python2.6/dist-packages" 
  /tmp/cephtest/enable-coredump /tmp/cephtest/binary/usr/local/bin/ceph-coverage /tmp/cephtest/archive/coverage
  /tmp/cephtest/workunit.client.0/rbd/watch_correct_version.sh'''
flavor: basic
mon.a-kernel-sha1: 694eba0efb52f4954e2a954cd2ed967167056245
mon.b-kernel-sha1: 694eba0efb52f4954e2a954cd2ed967167056245
owner: scheduled_teuthology@teuthology
success: false