Ceph

Logs: ubuntu@teuthology:/a/teuthology-2012-09-19_00:00:13-regression-next-testing-basic/25484

2012-09-19T02:52:33.354 INFO:teuthology.task.thrashosds.thrasher:starting do_thrash
2012-09-19T02:52:33.355 INFO:teuthology.task.thrashosds.thrasher:in_osds:  [0, 1, 2, 3, 4, 5]  out_osds:  [] dead_osds:  [] live_osds:  [1, 0, 3, 2, 5, 4]
2012-09-19T02:52:33.355 INFO:teuthology.task.thrashosds.thrasher:choose_action: min_in 2 min_out 0 min_live 2 min_dead 0
2012-09-19T02:52:33.355 INFO:teuthology.task.thrashosds.thrasher:Removing osd 1, in_osds are: [0, 1, 2, 3, 4, 5]
2012-09-19T02:52:33.355 DEBUG:teuthology.orchestra.run:Running: 'LD_LIBRARY_PRELOAD=/tmp/cephtest/binary/usr/local/lib /tmp/cephtest/enable-coredump /tmp/cephtest/binary/usr/local/bin/ceph-coverage /tmp/cephtest/archive/coverage /tmp/cephtest/binary/usr/local/bin/ceph -k /tmp/cephtest/ceph.keyring -c /tmp/cephtest/ceph.conf --concise osd out 1'
2012-09-19T02:52:33.360 INFO:teuthology.run_tasks:Running task admin_socket...
2012-09-19T02:52:33.361 DEBUG:teuthology.task.admin_socket:Running admin socket tests on client.0
2012-09-19T02:52:33.361 DEBUG:teuthology.orchestra.run:Running: "mkdir -- /tmp/cephtest/admin_socket_client.0 && while test '!' -e /tmp/cephtest/asok.client.0 ; do sleep 1 ; done" 
2012-09-19T02:52:33.387 INFO:teuthology.task.radosbench.radosbench.0.err:2012-09-19 02:52:33.369256 7f84dc2d3700  0 -- 10.214.133.32:0/1012643 >> 10.214.133.30:6789/0 pipe(0x2d1c410 sd=7 :55936 pgs=24 cs=1 l=1).injecting socket failure
2012-09-19T02:52:34.387 DEBUG:teuthology.task.admin_socket:Testing objecter_requests with config {'test': 'http://ceph.newdream.net/git/?p=ceph.git;a=blob_plain;f=src/test/admin_socket/objecter_requests;hb=HEAD'}
2012-09-19T02:52:34.387 DEBUG:teuthology.orchestra.run:Running: "wget -q -O /tmp/cephtest/admin_socket_client.0/objecter_requests -- 'http://ceph.newdream.net/git/?p=ceph.git;a=blob_plain;f=src/test/admin_socket/objecter_requests;hb=HEAD' && chmod u=rx -- /tmp/cephtest/admin_socket_client.0/objecter_requests" 
2012-09-19T02:52:34.759 DEBUG:teuthology.orchestra.run:Running: 'LD_LIBRARY_PATH=/tmp/cephtest/binary/usr/local/lib /tmp/cephtest/enable-coredump /tmp/cephtest/binary/usr/local/bin/ceph-coverage /tmp/cephtest/archive/coverage /tmp/cephtest/binary/usr/local/bin/ceph -k /tmp/cephtest/ceph.keyring -c /tmp/cephtest/ceph.conf --admin-daemon /tmp/cephtest/asok.client.0 objecter_requests'
2012-09-19T02:52:34.780 INFO:teuthology.task.radosbench.radosbench.0.err:2012-09-19 02:52:34.763161 7f84ddad6700 -1 asok(0x2d0ecb0) AdminSocket: request 'objecter_requests' not defined
2012-09-19T02:52:34.781 INFO:teuthology.orchestra.run.err:read only got 0 bytes of 4 expected for response length; invalid command?
2012-09-19T02:52:34.782 DEBUG:teuthology.task.admin_socket:admin socket command objecter_requests returned
2012-09-19T02:52:34.782 DEBUG:teuthology.orchestra.run:Running: 'rm -rf -- /tmp/cephtest/admin_socket_client.0'
2012-09-19T02:52:34.824 ERROR:teuthology.run_tasks:Saw exception from tasks
Traceback (most recent call last):
  File "/var/lib/teuthworker/teuthology/teuthology/run_tasks.py", line 25, in run_tasks
    manager = _run_one_task(taskname, ctx=ctx, config=config)
  File "/var/lib/teuthworker/teuthology/teuthology/run_tasks.py", line 14, in _run_one_task
    return fn(**kwargs)
  File "/var/lib/teuthworker/teuthology/teuthology/task/admin_socket.py", line 63, in task
    p.spawn(_run_tests, ctx, client, tests)
  File "/var/lib/teuthworker/teuthology/teuthology/parallel.py", line 83, in __exit__
    for result in self:
  File "/var/lib/teuthworker/teuthology/teuthology/parallel.py", line 100, in next
    resurrect_traceback(result)
  File "/var/lib/teuthworker/teuthology/teuthology/parallel.py", line 19, in capture_traceback
    return func(*args, **kwargs)
  File "/var/lib/teuthworker/teuthology/teuthology/task/admin_socket.py", line 140, in _run_tests
    sock_out = _socket_command(remote, socket_path, command, args)
  File "/var/lib/teuthworker/teuthology/teuthology/task/admin_socket.py", line 87, in _socket_command
    return json.loads(out)
  File "/usr/lib/python2.6/json/__init__.py", line 307, in loads
    return _default_decoder.decode(s)
  File "/usr/lib/python2.6/json/decoder.py", line 319, in decode
    obj, end = self.raw_decode(s, idx=_w(s, 0).end())
  File "/usr/lib/python2.6/json/decoder.py", line 338, in raw_decode
    raise ValueError("No JSON object could be decoded")
ValueError: No JSON object could be decoded
ubuntu@teuthology:/a/teuthology-2012-09-19_00:00:13-regression-next-testing-basic/25484$ cat config.yaml 
kernel: &id001
  kdb: true
  sha1: bac97e12668a12141fdc521d1833f8a13c4ae8e6
nuke-on-error: true
overrides:
  ceph:
    conf:
      global:
        ms inject socket failures: 5000
    fs: ext4
    log-whitelist:
    - slow request
    sha1: 7a3d1e66ef557881f8a96d72ac2ded003481174e
  s3tests:
    branch: next
  workunit:
    sha1: 7a3d1e66ef557881f8a96d72ac2ded003481174e
roles:
- - mon.a
  - osd.0
  - osd.1
  - osd.2
- - mds.a
  - osd.3
  - osd.4
  - osd.5
- - client.0
targets:
  ubuntu@plana84.front.sepia.ceph.com: ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABAQDVUrheZJ8d1kxgSwSsqLI81gK4Ekaikp7iC/lWc4fLSbUgdIkf8Yu3cmOHmvyIcZMwQD16BxiKRzmnD/tKN5KwLgBAawXPcfUuWSnzJQzPy8yp7q4PKw3OfqyGFqGp3XJAUpRa7Mki51zWXiiYqyCZQZOqMHnU8FwmC4kPTmvdcXuzxDd/sUNxu+CQYTgXAXdv9s3MBqyAxF9h54Icw5H8+rvoQTvN/BC45Sgg6KefnxmBq9gL9jzg8c+58bWno0wRs5fEerOcvYqObyiRA8Xvv/lS76G/u/rymDknvzmu0Tizw4wmoK2KpQ3f4Jbnhk9ntl+dBt0UXieTkcgJh4sv
  ubuntu@plana85.front.sepia.ceph.com: ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABAQDCQQz4HH+uZV5duWKnH+ZVOyBzgUIw3K4G31TlhhiDrxLQJIBFeYHacS9+TzSdTSsgDvikBmX43J/VUdEE+W6jL8bZ5cwXcULXdiaO6eFbnZOld76WdI3Rb6+F2cAQUBAFEmy4FvdGVteMYlLZVo+3VvMckM1C09RDlhA8AHyNf6AisZEkV0gu6ba7BKmfj8CxGI+Hs/zFRa0wyLmJgJIgPqvo5yikBAAZmVPyfOt8jUe3sfhf5R8o2hlr8ucBc9jHlpbspc1fJJ0HMOE2cIjCN5vup2CtjTmnxhghAeYujIJL8mkT81RiCPrXxngg8X8CkaH2KevviMElXjK/dtLL
  ubuntu@plana86.front.sepia.ceph.com: ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABAQDUMsCERn/r+eXKzlu9W/n6BFak26ReRaNUaccpQTrqOqhm3M7LiIGAeBo6JsygU9Kdtsm3115P4odiDbQLuNm0gKPw5DY7zDVqV+YRqe3kOrgIL/rxs6l6Y7htSRvzGhz7RsaG1fQH/BuaD18+s2WguKWwgRuuU1bvRYEHu0Y7qYYqUy+xJd4CGX+LMuAJzDYj5R7xIEYlJBln/c47He8q53cUUw5w48Hm47O8xo9ov7CqHCpOTZixeseY+zEm9skoLsUpBDIOmT+xUh7sOyKFhj+CnETpv3DocxGtoUgkx42GtkDhFK/dVV95Q6EOBseyYIUvFoHBKB2WV32xyxB5
tasks:
- internal.lock_machines: 3
- internal.save_config: null
- internal.check_lock: null
- internal.connect: null
- internal.check_conflict: null
- kernel: *id001
- internal.base: null
- internal.archive: null
- internal.coredump: null
- internal.syslog: null
- internal.timer: null
- chef: null
- clock: null
- ceph:
    log-whitelist:
    - wrongly marked me down
    - objects unfound and apparently lost
- thrashosds:
    timeout: 1200
- radosbench:
    clients:
    - client.0
    time: 60
- admin_socket:
    client.0:
      objecter_requests:
        test: http://ceph.newdream.net/git/?p=ceph.git;a=blob_plain;f=src/test/admin_socket/objecter_requests;hb=HEAD