Ceph

Logs are in http://qa-proxy.ceph.com/teuthology/teuthology-2014-10-05_10:00:04-upgrade:firefly-firefly-distro-basic-multi/528752/

2014-10-06T17:30:14.664 INFO:tasks.ceph:Waiting for all osds to be active and clean.
2014-10-06T17:30:24.657 INFO:tasks.ceph:Scrubbing terminated -- not all pgs were active and clean.
2014-10-06T17:30:24.658 ERROR:teuthology.contextutil:Saw exception from nested tasks
Traceback (most recent call last):
  File "/home/teuthworker/src/teuthology_master/teuthology/contextutil.py", line 29, in nested
    yield vars
  File "/var/lib/teuthworker/src/ceph-qa-suite_firefly/tasks/ceph.py", line 1230, in task
    yield
  File "/home/teuthworker/src/teuthology_master/teuthology/run_tasks.py", line 51, in run_tasks
    manager = run_one_task(taskname, ctx=ctx, config=config)
  File "/home/teuthworker/src/teuthology_master/teuthology/run_tasks.py", line 39, in run_one_task
    return fn(**kwargs)
  File "/home/teuthworker/src/teuthology_master/teuthology/task/sequential.py", line 55, in task
    mgr.__exit__(*exc_info)
  File "/usr/lib/python2.7/contextlib.py", line 24, in __exit__
    self.gen.next()
  File "/var/lib/teuthworker/src/ceph-qa-suite_firefly/tasks/thrashosds.py", line 178, in task
    thrash_proc.do_join()
  File "/var/lib/teuthworker/src/ceph-qa-suite_firefly/tasks/ceph_manager.py", line 165, in do_join
    self.thread.get()
  File "/usr/lib/python2.7/dist-packages/gevent/greenlet.py", line 308, in get
    raise self._exception
Exception: timed out waiting for admin_socket to appear after osd.4 restart

archive_path: /var/lib/teuthworker/archive/teuthology-2014-10-05_10:00:04-upgrade:firefly-firefly-distro-basic-multi/528752
branch: firefly
description: upgrade:firefly/newer/{0-cluster/start.yaml 1-install/v0.80.6.yaml 2-workload/blogbench.yaml
  3-upgrade-sequence/upgrade-mon-osd-mds.yaml 4-final/{monthrash.yaml osdthrash.yaml
  testrgw.yaml} distros/ubuntu_12.04.yaml}
email: ceph-qa@ceph.com
job_id: '528752'
kernel: &id001
  kdb: true
  sha1: distro
last_in_suite: false
machine_type: plana,mira,burnupi
name: teuthology-2014-10-05_10:00:04-upgrade:firefly-firefly-distro-basic-multi
nuke-on-error: true
os_type: ubuntu
os_version: '12.04'
overrides:
  admin_socket:
    branch: firefly
  ceph:
    conf:
      mon:
        debug mon: 20
        debug ms: 1
        debug paxos: 20
      osd:
        debug filestore: 20
        debug journal: 20
        debug ms: 1
        debug osd: 20
    fs: xfs
    log-whitelist:
    - slow request
    - scrub
    - wrongly marked me down
    - objects unfound and apparently lost
    - log bound mismatch
    sha1: 9a3bac0c1a7a42cdf7ba846c9ad0a3ae0f15b4bb
  ceph-deploy:
    branch:
      dev: firefly
    conf:
      client:
        log file: /var/log/ceph/ceph-$name.$pid.log
      mon:
        debug mon: 1
        debug ms: 20
        debug paxos: 20
        osd default pool size: 2
  install:
    ceph:
      sha1: 9a3bac0c1a7a42cdf7ba846c9ad0a3ae0f15b4bb
  s3tests:
    branch: firefly
  workunit:
    sha1: 9a3bac0c1a7a42cdf7ba846c9ad0a3ae0f15b4bb
owner: scheduled_teuthology@teuthology
priority: 1000
roles:
- - mon.a
  - mds.a
  - osd.0
  - osd.1
  - osd.2
- - mon.b
  - mon.c
  - osd.3
  - osd.4
  - osd.5
  - client.0
  - client.1
suite: upgrade:firefly
suite_branch: firefly
suite_path: /var/lib/teuthworker/src/ceph-qa-suite_firefly
targets:
  ubuntu@plana10.front.sepia.ceph.com: ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABAQDZM11QYmJVb8X/BHwLAOgozCK/ZG4XtzQZb5znJiFPwyi7vSZtCM7/DGWm6FGTUXg1bEcJHdD83EpvVgqTue/c6iTKCglzuB8eadW7gErwXjJgy/DCmJOHjmx/0fNCZi60nBVZQW3V1nptIgaFYHSFgPMhQSvEUg01ohfPlLKgnyeb6ciRxcDmJcrGpUrDNutxLf0aUC9gDF8tJ1nE5WV6tOSMNDpgVf9L3WwkxYj+G/77l+eSPqgDHs751C5iIzvt5eWTU+vfUd4uymPlBihPQXqnyfcriFJK8Tx80qofODJNxOLd9f+8wAUKXvY/pxSHL9h6sIrRI17TXPR9ip/V
  ubuntu@plana50.front.sepia.ceph.com: ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABAQCVWbmfptOM9BWnec9CUwypAVMmQj9hiPspeKJDAm5jY8ZImNEAS5KFNz+MPJraFvVryfLjZ9dHWDUTtBZHGWiirLCH8KeIZMwZSOUFJODzGbjN6VXUNAAWkRrEdSyksxHixsN6iyJa00D0K6rN5/7OgkJN36U+wsnavWlL+LJZlSd2KOQT7I3po2jSqqwZYqJ5WWMzwDTt+4x6N9mZ2dDvRiUEX7RPAfkG8+RDXoQiekG37EqAKNtI2oEkD7q/EpEOf9eATDFhrsTfyxHkvEIXz5mcpQqfus/JvDi+lUCjL0MQLYr9cCEVDTUJznXcq1EmvbmLpVNiBqqdDYsgOWlx
tasks:
- internal.lock_machines:
  - 2
  - plana,mira,burnupi
- internal.save_config: null
- internal.check_lock: null
- internal.connect: null
- internal.push_inventory: null
- internal.serialize_remote_roles: null
- internal.check_conflict: null
- internal.check_ceph_data: null
- internal.vm_setup: null
- kernel: *id001
- internal.base: null
- internal.archive: null
- internal.coredump: null
- internal.sudo: null
- internal.syslog: null
- internal.timer: null
- chef: null
- clock.check: null
- install:
    tag: v0.80.6
- ceph:
    log-whitelist:
    - scrub mismatch
    - ScrubResult
- parallel:
  - workload
  - upgrade-sequence
- sequential:
  - mon_thrash:
      revive_delay: 20
      thrash_delay: 1
  - ceph-fuse: null
  - workunit:
      clients:
        client.0:
        - suites/dbench.sh
- sequential:
  - thrashosds:
      chance_pgnum_grow: 1
      chance_pgpnum_fix: 1
      timeout: 1200
  - ceph-fuse:
    - client.0
  - workunit:
      clients:
        client.0:
        - suites/iogen.sh
- sequential:
  - rgw:
    - client.1
  - s3tests:
      client.1:
        rgw_server: client.1
teuthology_branch: master
tube: multi
upgrade-sequence:
  sequential:
  - install.upgrade:
      all:
        branch: firefly
  - ceph.restart:
    - mon.a
  - sleep:
      duration: 60
  - ceph.restart:
    - mon.b
  - sleep:
      duration: 60
  - ceph.restart:
    - mon.c
  - sleep:
      duration: 60
  - ceph.restart:
    - mds.a
  - sleep:
      duration: 60
  - ceph.restart:
    - osd.0
  - sleep:
      duration: 30
  - ceph.restart:
    - osd.1
  - sleep:
      duration: 30
  - ceph.restart:
    - osd.2
  - sleep:
      duration: 30
  - ceph.restart:
    - osd.3
  - sleep:
      duration: 30
  - ceph.restart:
    - osd.4
  - sleep:
      duration: 30
  - ceph.restart:
    - osd.5
verbose: true
worker_log: /var/lib/teuthworker/archive/worker_logs/worker.multi.3193
workload:
  workunit:
    clients:
      client.0:
      - suites/blogbench.sh

description: upgrade:firefly/newer/{0-cluster/start.yaml 1-install/v0.80.6.yaml 2-workload/blogbench.yaml
  3-upgrade-sequence/upgrade-mon-osd-mds.yaml 4-final/{monthrash.yaml osdthrash.yaml
  testrgw.yaml} distros/ubuntu_12.04.yaml}
duration: 3499.8261671066284
failure_reason: timed out waiting for admin_socket to appear after osd.4 restart
flavor: basic
owner: scheduled_teuthology@teuthology
success: false