Tools » teuthology

Logs are in http://qa-proxy.ceph.com/teuthology/teuthology-2014-09-20_15:08:15-upgrade:firefly-firefly-testing-basic-vps/500947/

2014-09-20T16:34:29.446 INFO:tasks.mon_thrash:joining mon_thrasher
2014-09-20T16:34:29.446 ERROR:teuthology.run_tasks:Saw exception from tasks.
Traceback (most recent call last):
  File "/home/teuthworker/src/teuthology_master/teuthology/run_tasks.py", line 51, in run_tasks
    manager = run_one_task(taskname, ctx=ctx, config=config)
  File "/home/teuthworker/src/teuthology_master/teuthology/run_tasks.py", line 39, in run_one_task
    return fn(**kwargs)
  File "/home/teuthworker/src/teuthology_master/teuthology/task/sequential.py", line 55, in task
    mgr.__exit__(*exc_info)
  File "/usr/lib/python2.7/contextlib.py", line 24, in __exit__
    self.gen.next()
  File "/var/lib/teuthworker/src/ceph-qa-suite_firefly/tasks/mon_thrash.py", line 341, in task
    thrash_proc.do_join()
  File "/var/lib/teuthworker/src/ceph-qa-suite_firefly/tasks/mon_thrash.py", line 143, in do_join
    self.thread.get()
  File "/usr/lib/python2.7/dist-packages/gevent/greenlet.py", line 308, in get
    raise self._exception
AssertionError

archive_path: /var/lib/teuthworker/archive/teuthology-2014-09-20_15:08:15-upgrade:firefly-firefly-testing-basic-vps/500947
branch: firefly
description: upgrade:firefly/newer/{0-cluster/start.yaml 1-install/v0.80.4.yaml 2-workload/s3tests.yaml
  3-upgrade-sequence/upgrade-osd-mon-mds.yaml 4-final/{monthrash.yaml osdthrash.yaml
  testrgw.yaml} distros/ubuntu_12.04.yaml}
email: ceph-qa@ceph.com
job_id: '500947'
kernel: &id001
  kdb: true
  sha1: 3ed0e301a4806e64958a50b28a34abc093b953ab
last_in_suite: false
machine_type: vps
name: teuthology-2014-09-20_15:08:15-upgrade:firefly-firefly-testing-basic-vps
nuke-on-error: true
os_type: ubuntu
os_version: '12.04'
overrides:
  admin_socket:
    branch: firefly
  ceph:
    conf:
      global:
        osd heartbeat grace: 100
      mon:
        debug mon: 20
        debug ms: 1
        debug paxos: 20
      osd:
        debug filestore: 20
        debug journal: 20
        debug ms: 1
        debug osd: 20
    fs: xfs
    log-whitelist:
    - slow request
    - scrub
    - wrongly marked me down
    - objects unfound and apparently lost
    - log bound mismatch
    sha1: 2675b0d159ff0608bc6e607be3727ed23e7e2976
  ceph-deploy:
    branch:
      dev: firefly
    conf:
      client:
        log file: /var/log/ceph/ceph-$name.$pid.log
      mon:
        debug mon: 1
        debug ms: 20
        debug paxos: 20
        osd default pool size: 2
  install:
    ceph:
      sha1: 2675b0d159ff0608bc6e607be3727ed23e7e2976
  rgw:
    default_idle_timeout: 1200
  s3tests:
    branch: firefly
    idle_timeout: 1200
  workunit:
    sha1: 2675b0d159ff0608bc6e607be3727ed23e7e2976
owner: scheduled_teuthology@teuthology
priority: 1000
roles:
- - mon.a
  - mds.a
  - osd.0
  - osd.1
  - osd.2
- - mon.b
  - mon.c
  - osd.3
  - osd.4
  - osd.5
  - client.0
  - client.1
suite: upgrade:firefly
suite_branch: firefly
suite_path: /var/lib/teuthworker/src/ceph-qa-suite_firefly
targets:
  ubuntu@vpm095.front.sepia.ceph.com: ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABAQCnlT1qHmoqX2CY7ZwzNFMQ92QwIpkEOkEQut7MrU/A3pgE1+7OzUS3di+Ll4OXTZ8nUbEck0FDfypz6VA7NUBfX8o5hIL+Py+geuSR3y1Hqm8H6l7Np6BwZLlsyZYB7NtOOVlmpdINb3gXlEgfNJ6uV11312LJcQ3w8XRNGgDxI9akrLof8mxeHYPsm8JS06JL62Sl1emrkY/G2TxLtzwmcOLSCDMiZDY8PKZml2hhDqgU90NcAoQA5JGgXY46HFG6xtfLbXc51A6AVGWGKvY8rme16ILaCGUop7JjM8OeA+wGBB/npqV2OWTvHknQOjOZtRALZXoPblRaWta8KY7Z
  ubuntu@vpm181.front.sepia.ceph.com: ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABAQDW9wc78t/XYdz5twDBDn9/sxJzVE2YkHZUAckC7tEspmNTcKiSTTGZukXPqeXl9fEUxg9Dr30HE+lCqeDND6lirsyaouKbY2pW2+yx+Mn1G11pYmPXEuxrq4fXWaLzb4dFZOVM82S6UNz0XbLBsySQ6iMxT4ITlHdI20mXXyXxeW75NTGm1TUU4ijOlo7DLOQJnVFlq4Yp8+hoqaUn4OG7ISVXF/aEKZ2BRz+DgF2czvJ6FGDJpU3QOfGLnnOmGq1RkqdWPHNiAI+PBGpkElCfviPwqmNt0ouhKcVRPl9MQCJbV0GkmQ2V/3GBQvnEpJP5Nf4cmehrbR/uunWs1MVt
tasks:
- internal.lock_machines:
  - 2
  - vps
- internal.save_config: null
- internal.check_lock: null
- internal.connect: null
- internal.push_inventory: null
- internal.serialize_remote_roles: null
- internal.check_conflict: null
- internal.check_ceph_data: null
- internal.vm_setup: null
- kernel: *id001
- internal.base: null
- internal.archive: null
- internal.coredump: null
- internal.sudo: null
- internal.syslog: null
- internal.timer: null
- chef: null
- clock.check: null
- install:
    tag: v0.80.4
- ceph:
    log-whitelist:
    - scrub mismatch
    - ScrubResult
- parallel:
  - workload
  - upgrade-sequence
- sequential:
  - mon_thrash:
      revive_delay: 20
      thrash_delay: 1
  - ceph-fuse: null
  - workunit:
      clients:
        client.0:
        - suites/dbench.sh
- sequential:
  - thrashosds:
      chance_pgnum_grow: 1
      chance_pgpnum_fix: 1
      timeout: 1200
  - ceph-fuse:
    - client.0
  - workunit:
      clients:
        client.0:
        - suites/iogen.sh
- sequential:
  - rgw:
    - client.1
  - s3tests:
      client.1:
        rgw_server: client.1
teuthology_branch: master
tube: vps
upgrade-sequence:
  sequential:
  - install.upgrade:
      all:
        branch: firefly
  - ceph.restart:
    - osd.0
  - sleep:
      duration: 30
  - ceph.restart:
    - osd.1
  - sleep:
      duration: 30
  - ceph.restart:
    - osd.2
  - sleep:
      duration: 30
  - ceph.restart:
    - osd.3
  - sleep:
      duration: 30
  - ceph.restart:
    - osd.4
  - sleep:
      duration: 30
  - ceph.restart:
    - osd.5
  - sleep:
      duration: 60
  - ceph.restart:
    - mon.a
  - sleep:
      duration: 60
  - ceph.restart:
    - mon.b
  - sleep:
      duration: 60
  - ceph.restart:
    - mon.c
  - sleep:
      duration: 60
  - ceph.restart:
    - mds.a
verbose: true
worker_log: /var/lib/teuthworker/archive/worker_logs/worker.vps.3040
workload:
  sequential:
  - rgw:
    - client.0
  - s3tests:
      client.0:
        force-branch: firefly-original
        rgw_server: client.0

description: upgrade:firefly/newer/{0-cluster/start.yaml 1-install/v0.80.4.yaml 2-workload/s3tests.yaml
  3-upgrade-sequence/upgrade-osd-mon-mds.yaml 4-final/{monthrash.yaml osdthrash.yaml
  testrgw.yaml} distros/ubuntu_12.04.yaml}
duration: 3223.2652819156647
failure_reason: ''
flavor: basic
mon.a-kernel-sha1: 3ed0e301a4806e64958a50b28a34abc093b953ab
mon.b-kernel-sha1: 3ed0e301a4806e64958a50b28a34abc093b953ab
owner: scheduled_teuthology@teuthology
success: false