Ceph » rgw

Logs are in qa-proxy.ceph.com/teuthology/teuthology-2014-02-22_23:02:21-rgw-firefly-distro-basic-plana/98483

Error:

2014-02-23T20:45:20.615 DEBUG:teuthology.orchestra.run:Running [10.214.132.15]: "sudo egrep '\\[ERR\\]|\\[WRN\\]|\\[SEC\\]' /var/log/ceph/ceph.log | egrep -v 'slow request' | head -n 1" 
2014-02-23T20:45:20.674 WARNING:teuthology.task.ceph:Found errors (ERR|WRN|SEC) in cluster log
2014-02-23T20:45:20.675 DEBUG:teuthology.orchestra.run:Running [10.214.132.15]: "sudo egrep '\\[SEC\\]' /var/log/ceph/ceph.log | egrep -v 'slow request' | head -n 1" 
2014-02-23T20:45:20.749 DEBUG:teuthology.orchestra.run:Running [10.214.132.15]: "sudo egrep '\\[ERR\\]' /var/log/ceph/ceph.log | egrep -v 'slow request' | head -n 1" 
2014-02-23T20:45:20.824 INFO:teuthology.task.ceph:Unmounting /var/lib/ceph/osd/ceph-0 on ubuntu@plana63.front.sepia.ceph.com
2014-02-23T20:45:20.824 DEBUG:teuthology.orchestra.run:Running [10.214.132.15]: 'sync && sudo umount -f /var/lib/ceph/osd/ceph-0'

2014-02-23T20:49:38.410 INFO:teuthology.run:Summary data:
{description: 'rgw/multifs/{clusters/fixed-2.yaml fs/btrfs.yaml tasks/rgw_readwrite.yaml}',
  duration: 1354.3043611049652, failure_reason: '"2014-02-23 20:44:10.145377 osd.2
    10.214.132.15:6810/8004 18 : [ERR] 12.4 scrub stat mismatch, got 279/279 objects,
    0/0 clones, 279/35 dirty, 0/0 whiteouts, 398208541/398208541 bytes." in cluster
    log', flavor: basic, owner: scheduled_teuthology@teuthology, success: false}

YAML config:

archive_path: /var/lib/teuthworker/archive/teuthology-2014-02-22_23:02:21-rgw-firefly-distro-basic-plana/98483
description: rgw/multifs/{clusters/fixed-2.yaml fs/btrfs.yaml tasks/rgw_readwrite.yaml}
email: null
job_id: '98483'
kernel: &id001
  kdb: true
  sha1: distro
last_in_suite: false
machine_type: plana
name: teuthology-2014-02-22_23:02:21-rgw-firefly-distro-basic-plana
nuke-on-error: true
os_type: ubuntu
overrides:
  admin_socket:
    branch: firefly
  ceph:
    conf:
      mon:
        debug mon: 20
        debug ms: 1
        debug paxos: 20
      osd:
        debug filestore: 20
        debug ms: 1
        debug osd: 20
        osd op thread timeout: 60
        osd sloppy crc: true
    fs: btrfs
    log-whitelist:
    - slow request
    sha1: 6d8cb22e5887b3289bf3e0809f207442874aaccc
  ceph-deploy:
    branch:
      dev: firefly
    conf:
      client:
        log file: /var/log/ceph/ceph-$name.$pid.log
      mon:
        debug mon: 1
        debug ms: 20
        debug paxos: 20
        osd default pool size: 2
  install:
    ceph:
      sha1: 6d8cb22e5887b3289bf3e0809f207442874aaccc
  s3tests:
    branch: master
  workunit:
    sha1: 6d8cb22e5887b3289bf3e0809f207442874aaccc
owner: scheduled_teuthology@teuthology
roles:
- - mon.a
  - mon.c
  - osd.0
  - osd.1
  - osd.2
  - client.0
- - mon.b
  - mds.a
  - osd.3
  - osd.4
  - osd.5
  - client.1
targets:
  ubuntu@plana62.front.sepia.ceph.com: ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABAQDSC3e8HvnLeaoPWNNoKjTmvf5ebfvXKGCgTNCjPeI9Qtl9pnLXo5XniCH6tTFWpYMQLtesPysgoQxCHEOEzui/YHwXfq3JPWIPMlANLNBH1e6RRqjHC+mIzSNr0nP295AUFZycyAIxRDRUWwz8IWaPtEjqohd+oojP/uBklTaHyDEZ84nEabMXy1giwPwjQeEYrTNBtuTDTkOZryrrP6XxoBEKK/P3ZyD54ouSdUb2K3XVzVqrNdf4+pUh6TuctCPpbmewEuW4tygAPZhaNmHujMkhT7DsfIcTFR/dDig0ubYlGExyiOsrXF1eRHLKGPMYl6Y7ZFTLOOkD1OBVeJ8/
  ubuntu@plana63.front.sepia.ceph.com: ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABAQDXFu3H/FZPa959qQb84rw3UusSzxbN1gK+JkuL7U6tL4ebmbFpEGOigEQM+qUIPcjiQw8/AJDZ17WbnQfZ90Ffv0MCoSsYRq2AefBOPaUcEuCj/QhoI+3E7a6XU/HSrA9V02EIur2Oe3cZAX6tv1PDylNLC5cTpWHr5LGO/DsNEKKhMPjCyfzBzGDTuo4sxtcK8d370Y+QHiJPtjJAbYkb/DeqT2py8ST6N8qWZgDJK2zDr4OxWz90GkBgCxWdCE0/D/hs4k82PDfB2Zd2CIpkl4CB96wzavBzrgr9wiVotrAEJFtxNDYHMLtbRg9d8GzxbcjxH8OkxOsgsgRB1SKj
tasks:
- internal.lock_machines:
  - 2
  - plana
- internal.save_config: null
- internal.check_lock: null
- internal.connect: null
- internal.check_conflict: null
- internal.check_ceph_data: null
- internal.vm_setup: null
- kernel: *id001
- internal.base: null
- internal.archive: null
- internal.coredump: null
- internal.sudo: null
- internal.syslog: null
- internal.timer: null
- chef: null
- clock.check: null
- install: null
- ceph: null
- rgw:
  - client.0
- s3readwrite:
    client.0:
      readwrite:
        bucket: rwtest
        duration: 300
        files:
          num: 10
          size: 2000
          stddev: 500
        readers: 10
        writers: 3
      rgw_server: client.0
teuthology_branch: firefly
verbose: true
worker_log: /var/lib/teuthworker/archive/worker_logs/worker.plana.3317

Summary:

description: rgw/multifs/{clusters/fixed-2.yaml fs/btrfs.yaml tasks/rgw_readwrite.yaml}
duration: 1354.3043611049652
failure_reason: '"2014-02-23 20:44:10.145377 osd.2 10.214.132.15:6810/8004 18 : [ERR]
  12.4 scrub stat mismatch, got 279/279 objects, 0/0 clones, 279/35 dirty, 0/0 whiteouts,
  398208541/398208541 bytes." in cluster log'
flavor: basic
owner: scheduled_teuthology@teuthology
success: false