Ceph

Logs are in qa-proxy.ceph.com/teuthology/teuthology-2014-03-05_10:33:26-upgrade:dumpling-x:parallel-firefly---basic-plana/117795/

2014-03-05T21:30:27.673 INFO:teuthology.task.workunit.client.1.err:[10.214.131.3]: + ceph_test_rados_api_tier
2014-03-05T21:30:27.673 INFO:teuthology.task.workunit.client.1.err:[10.214.131.3]: /home/ubuntu/cephtest/workunit.client.1/rados/test.sh: 8: /home/ubuntu/cephtest/workunit.client.1/rados/test.sh: ceph_test_rados_api_tier: not found
2014-03-05T21:30:27.674 INFO:teuthology.task.workunit:Stopping rados/test.sh on client.1...
2014-03-05T21:30:27.674 DEBUG:teuthology.orchestra.run:Running [10.214.131.3]: 'rm -rf -- /home/ubuntu/cephtest/workunits.list /home/ubuntu/cephtest/workunit.client.1'
2014-03-05T21:30:27.684 ERROR:teuthology.parallel:Exception in parallel execution
Traceback (most recent call last):
  File "/home/teuthworker/teuthology-firefly/teuthology/parallel.py", line 82, in __exit__
    for result in self:
  File "/home/teuthworker/teuthology-firefly/teuthology/parallel.py", line 101, in next
    resurrect_traceback(result)
  File "/home/teuthworker/teuthology-firefly/teuthology/parallel.py", line 19, in capture_traceback
    return func(*args, **kwargs)
  File "/home/teuthworker/teuthology-firefly/teuthology/task/workunit.py", line 359, in _run_tests
    args=args,
  File "/home/teuthworker/teuthology-firefly/teuthology/orchestra/remote.py", line 106, in run
    r = self._runner(client=self.ssh, **kwargs)
  File "/home/teuthworker/teuthology-firefly/teuthology/orchestra/run.py", line 328, in run
    r.exitstatus = _check_status(r.exitstatus)
  File "/home/teuthworker/teuthology-firefly/teuthology/orchestra/run.py", line 324, in _check_status
    raise CommandFailedError(command=r.command, exitstatus=status, node=host)
CommandFailedError: Command failed on 10.214.131.3 with status 127: 'mkdir -p -- /home/ubuntu/cephtest/mnt.1/client.1/tmp && cd -- /home/ubuntu/cephtest/mnt.1/client.1/tmp && CEPH_CLI_TEST_DUP_COMMAND=1 CEPH_REF=4901347e893623900746e056dfd72586edf05fef TESTDIR="/home/ubuntu/cephtest" CEPH_ID="1" adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 6h /home/ubuntu/cephtest/workunit.client.1/rados/test.sh'
2014-03-05T21:30:27.685 ERROR:teuthology.run_tasks:Saw exception from tasks.
Traceback (most recent call last):
  File "/home/teuthworker/teuthology-firefly/teuthology/run_tasks.py", line 31, in run_tasks
    manager = run_one_task(taskname, ctx=ctx, config=config)
  File "/home/teuthworker/teuthology-firefly/teuthology/run_tasks.py", line 19, in run_one_task
    return fn(**kwargs)
  File "/home/teuthworker/teuthology-firefly/teuthology/task/workunit.py", line 102, in task
    all_spec = True
  File "/home/teuthworker/teuthology-firefly/teuthology/parallel.py", line 82, in __exit__
    for result in self:
  File "/home/teuthworker/teuthology-firefly/teuthology/parallel.py", line 101, in next
    resurrect_traceback(result)
  File "/home/teuthworker/teuthology-firefly/teuthology/parallel.py", line 19, in capture_traceback
    return func(*args, **kwargs)
  File "/home/teuthworker/teuthology-firefly/teuthology/task/workunit.py", line 359, in _run_tests
    args=args,
  File "/home/teuthworker/teuthology-firefly/teuthology/orchestra/remote.py", line 106, in run
    r = self._runner(client=self.ssh, **kwargs)
  File "/home/teuthworker/teuthology-firefly/teuthology/orchestra/run.py", line 328, in run
    r.exitstatus = _check_status(r.exitstatus)
  File "/home/teuthworker/teuthology-firefly/teuthology/orchestra/run.py", line 324, in _check_status
    raise CommandFailedError(command=r.command, exitstatus=status, node=host)
CommandFailedError: Command failed on 10.214.131.3 with status 127: 'mkdir -p -- /home/ubuntu/cephtest/mnt.1/client.1/tmp && cd -- /home/ubuntu/cephtest/mnt.1/client.1/tmp && CEPH_CLI_TEST_DUP_COMMAND=1 CEPH_REF=4901347e893623900746e056dfd72586edf05fef TESTDIR="/home/ubuntu/cephtest" CEPH_ID="1" adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 6h /home/ubuntu/cephtest/workunit.client.1/rados/test.sh'
2014-03-05T21:30:27.717 ERROR:teuthology.run_tasks: Sentry event: http://sentry.ceph.com/inktank/teuthology/search?q=86820bb27d4e40529c02050ff4421c0f
CommandFailedError: Command failed on 10.214.131.3 with status 127: 'mkdir -p -- /home/ubuntu/cephtest/mnt.1/client.1/tmp && cd -- /home/ubuntu/cephtest/mnt.1/client.1/tmp && CEPH_CLI_TEST_DUP_COMMAND=1 CEPH_REF=4901347e893623900746e056dfd72586edf05fef TESTDIR="/home/ubuntu/cephtest" CEPH_ID="1" adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 6h /home/ubuntu/cephtest/workunit.client.1/rados/test.sh'
2014-03-05T21:30:27.717 DEBUG:teuthology.run_tasks:Unwinding manager mon_thrash
2014-03-05T21:30:27.717 INFO:teuthology.task.mon_thrash:joining mon_thrasher

archive_path: /var/lib/teuthworker/archive/teuthology-2014-03-05_10:33:26-upgrade:dumpling-x:parallel-firefly---basic-plana/117795
description: upgrade/dumpling-x/parallel/{0-cluster/start.yaml 1-dumpling-install/dumpling.yaml
  2-workload/rados_api.yaml 3-upgrade-sequence/upgrade-all.yaml 4-final-workload/rados_mon_thrash.yaml
  distros/ubuntu_12.04.yaml}
email: null
job_id: '117795'
last_in_suite: false
machine_type: plana
name: teuthology-2014-03-05_10:33:26-upgrade:dumpling-x:parallel-firefly---basic-plana
nuke-on-error: true
os_type: ubuntu
os_version: '12.04'
overrides:
  admin_socket:
    branch: firefly
  ceph:
    conf:
      mon:
        debug mon: 20
        debug ms: 1
        debug paxos: 20
      osd:
        debug filestore: 20
        debug ms: 1
        debug osd: 20
    log-whitelist:
    - slow request
    sha1: 4901347e893623900746e056dfd72586edf05fef
  ceph-deploy:
    branch:
      dev: firefly
    conf:
      client:
        log file: /var/log/ceph/ceph-$name.$pid.log
      mon:
        debug mon: 1
        debug ms: 20
        debug paxos: 20
        osd default pool size: 2
  install:
    ceph:
      sha1: 4901347e893623900746e056dfd72586edf05fef
  s3tests:
    branch: master
  workunit:
    sha1: 4901347e893623900746e056dfd72586edf05fef
owner: scheduled_teuthology@teuthology
roles:
- - mon.a
  - mds.a
  - osd.0
  - osd.1
- - mon.b
  - mon.c
  - osd.2
  - osd.3
- - client.0
  - client.1
targets:
  ubuntu@plana35.front.sepia.ceph.com: ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABAQDN2DvVrroCqjz4NeYHoqEhj35Ea5bEEMc2YHJY0rfXO0J9cRSnQBfzOQtSC6LrMYXJrQ4Wr/UQmmovs6a42F/fVmYygUeCdAjy6RT5bQ7eHdd+bJB3zKSZ+oY72g8UHlHRekSFBDI8ivFLC7PWscUx4v1o5vgQFgUDXkBjZpje7VpB3Sp8P/Dpqf5zcEo/9DxYxntaxn3dg2LrGZ4jKG5A49ivCo6NOxFRyth3dlcTMlhPjVs6dKWzohppx9h6bW5k0ijF3FLfPfc72Nkdjiw8W3njDbXl8JjKTvMrfKc1wxDEeq7602oIuT7CFC4hrBZtxLPinrtQhguXB0j35git
  ubuntu@plana37.front.sepia.ceph.com: ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABAQC8FPsPKV1KVlb89QL2k0kNMTM3mIenC2wHxnVb9EgA7MGjC/gJFv4FoYFtTn0SadJl2hZNJ8kk7HjBsgCQG3f+LL3l7DPlqSJG8zFFXW6LCzjk0YQX/JX7X6nK33HdxzzOZVecglaQnTSWKbPDp8ofd9EQX4gN7mPb/C0/FUtT0Hjrb97QBYqDDVWEMBo7BCT4YdsisPBkCFpQ1Khl2K89e9uhfw4wvVvqveLnU3NEAULbEhMeLg0LMsSlmK2gfiyJbyxweApXo4VqfuNd6DnUqUzilAM0VJL3KgJqJGW46IYC76VPMSHPKD66kgrYiyBm12iLEy70kODNVaNe3wnX
  ubuntu@plana38.front.sepia.ceph.com: ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABAQDXzZ48SY1LqesuHF5s+wz8siV1ptIMhbU9GUSH7VGII/TqNT5drF0rs+9hhFh1y9VspjqX9ZIVCy2k2QPVQpQnBMEls/JO3NJfJbcC+YbWoiuW3xm+IlaDhtlSXkSN399PzV55IgPrL32gtudyhGJvpLLmDt1VdMUBAK4j0b2uPGojiR+yKklUb1aOs88/4s45r0gwpmNmeFo86hD9b4MkY85b1+Fn73c/tyOieEClsxtivgf45mQ4UkIezkgvZfk18q/lEYWzZWZATtK/FyAfzYNVRn3l9IGaxnP9z3lZS9irAJEYAYxW2BrA4pRMYvtHojctKXpRCBQBV+Wuk0aN
tasks:
- internal.lock_machines:
  - 3
  - plana
- internal.save_config: null
- internal.check_lock: null
- internal.connect: null
- internal.check_conflict: null
- internal.check_ceph_data: null
- internal.vm_setup: null
- internal.base: null
- internal.archive: null
- internal.coredump: null
- internal.sudo: null
- internal.syslog: null
- internal.timer: null
- chef: null
- clock.check: null
- install:
    branch: dumpling
- ceph:
    fs: xfs
- parallel:
  - workload
  - upgrade-sequence
- mon_thrash:
    revive_delay: 20
    thrash_delay: 1
- workunit:
    clients:
      client.1:
      - rados/test.sh
teuthology_branch: firefly
upgrade-sequence:
  sequential:
  - install.upgrade: null
  - ceph.restart:
    - mon.a
    - mon.b
    - mon.c
    - mds.a
    - osd.0
    - osd.1
    - osd.2
    - osd.3
verbose: true
worker_log: /var/lib/teuthworker/archive/worker_logs/worker.plana.11129
workload:
  sequential:
  - branch: dumpling
    clients:
      client.0:
      - rados/test.sh
      - cls
    workunit: null

description: upgrade/dumpling-x/parallel/{0-cluster/start.yaml 1-dumpling-install/dumpling.yaml
  2-workload/rados_api.yaml 3-upgrade-sequence/upgrade-all.yaml 4-final-workload/rados_mon_thrash.yaml
  distros/ubuntu_12.04.yaml}
duration: 925.6273448467255
failure_reason: 'Command failed on 10.214.131.3 with status 127: ''mkdir -p -- /home/ubuntu/cephtest/mnt.1/client.1/tmp
  && cd -- /home/ubuntu/cephtest/mnt.1/client.1/tmp && CEPH_CLI_TEST_DUP_COMMAND=1
  CEPH_REF=4901347e893623900746e056dfd72586edf05fef TESTDIR="/home/ubuntu/cephtest" 
  CEPH_ID="1" adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage
  timeout 6h /home/ubuntu/cephtest/workunit.client.1/rados/test.sh'''
flavor: basic
owner: scheduled_teuthology@teuthology
sentry_event: http://sentry.ceph.com/inktank/teuthology/search?q=86820bb27d4e40529c02050ff4421c0f
success: false