$ PYTHONPATH=/home/loic/software/ceph/ceph-qa-suite ./virtualenv/bin/teuthology --verbose --owner loic@dachary.org ../ceph-qa-suite/suites/rados/singleton/all/ec-lost-unfound.yaml targets.yaml 2014-09-25 21:43:40,447.447 WARNING:teuthology.report:No job_id found; not reporting results 2014-09-25 21:43:40,449.449 DEBUG:teuthology.run:Config: roles: - - mon.a - mon.b - mon.c - osd.0 - osd.1 - osd.2 - osd.3 targets: vpm057.front.sepia.ceph.com: ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABAQDAQdd/DacyAgqtNZxMPxkWsrNy2+veFyTqbUCgeRzaQWHYfKg09mYqFXPwR1qlguH+H14JOtTDAF2iR+hJUd8fMSFx/NMEi4++eXdbC/lHWll3p9xHKvi9ac2bhACyb3l7Kz4ojUMrPFjt6jddVgHp09jwMXaV0ugj2lmhQe+SQePZLmCDmgfZA/xq5LllxtOBw4aOONlhVHa9W0BB8cX3B/QK3VuxrF+BzTqYAj+7m7M5osERp6ykstojJ4yocdHl2MAweO8umb89jNmH+3HG0QVNQqOJidsvajvgKMaQRHP7dtUImhJmY5EVD0bXVzb5r+gAKr5c8RuMLxVms2OJ tasks: - install: null - ceph: log-whitelist: - objects unfound and apparently lost - ec_lost_unfound: null 2014-09-25 21:43:40,449.449 INFO:teuthology.run:Found tasks at /home/loic/software/ceph/ceph-qa-suite/tasks 2014-09-25 21:43:40,449.449 INFO:teuthology.run_tasks:Running task internal.save_config... 2014-09-25 21:43:40,449.449 INFO:teuthology.task.internal:Saving configuration 2014-09-25 21:43:40,449.449 INFO:teuthology.run_tasks:Running task internal.check_lock... 2014-09-25 21:43:40,449.449 INFO:teuthology.task.internal:Checking locks... 2014-09-25 21:43:40,451.451 INFO:urllib3.connectionpool:Starting new HTTP connection (1): paddles.front.sepia.ceph.com 2014-09-25 21:43:41,400.400 DEBUG:teuthology.task.internal:machine status is {u'is_vm': True, u'locked': True, u'locked_since': u'2014-09-25 18:40:22.380610', u'locked_by': u'loic@dachary.org', u'up': True, u'mac_address': u'52:54:00:96:29:a3', u'name': u'vpm057.front.sepia.ceph.com', u'os_version': u'14.04', u'machine_type': u'vps', u'vm_host': {u'is_vm': False, u'locked': True, u'locked_since': u'2013-03-14 19:29:52', u'locked_by': u'VPSHOST@VPSHOST', u'up': True, u'mac_address': u'00:25:90:00:57:ea', u'name': u'mira010.front.sepia.ceph.com', u'os_version': None, u'machine_type': u'mira', u'vm_host': None, u'os_type': u'ubuntu', u'arch': u'x86_64', u'ssh_pub_key': u'ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABAQDb8fuH6089eXGdxAPSHf7ve/8dpbKS5Mswi9H3/SnHROmzmIuBlaM/vVArXorniAkz2T7p4zTLCzQLuPAqzVkvmKP+e5eB3EFZ818pHwQ/AcmvPEi65DZ5Y2u406HMaH70ig2HUjI9hyYVbB+WMTYtTZi/3PGNVk/uUOQvH7WjpVA08b6jhqHi9DqN3FqsSXOfNFA1c07jmrZ6vz8xOkGIZt40DhObUDNOMJ7YxAlxtw6OomW+xHtCCgtCjLFlgyTEwPcl/TGlDFAFDadFQpR3Ikx5c3Eqvpq0SnH/KqdkWv8FDBfG8QcTnG55e1bZBuPvQMZXjfrKvBcsY1hbGwCh', u'description': u''}, u'os_type': u'ubuntu', u'arch': u'x86_64', u'ssh_pub_key': u'ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABAQDAQdd/DacyAgqtNZxMPxkWsrNy2+veFyTqbUCgeRzaQWHYfKg09mYqFXPwR1qlguH+H14JOtTDAF2iR+hJUd8fMSFx/NMEi4++eXdbC/lHWll3p9xHKvi9ac2bhACyb3l7Kz4ojUMrPFjt6jddVgHp09jwMXaV0ugj2lmhQe+SQePZLmCDmgfZA/xq5LllxtOBw4aOONlhVHa9W0BB8cX3B/QK3VuxrF+BzTqYAj+7m7M5osERp6ykstojJ4yocdHl2MAweO8umb89jNmH+3HG0QVNQqOJidsvajvgKMaQRHP7dtUImhJmY5EVD0bXVzb5r+gAKr5c8RuMLxVms2OJ', u'description': None} 2014-09-25 21:43:41,400.400 INFO:teuthology.run_tasks:Running task internal.connect... 2014-09-25 21:43:41,400.400 INFO:teuthology.task.internal:Opening connections... 2014-09-25 21:43:41,400.400 DEBUG:teuthology.task.internal:connecting to ubuntu@vpm057.front.sepia.ceph.com 2014-09-25 21:43:41,404.404 INFO:teuthology.orchestra.connection:{'username': 'ubuntu', 'hostname': 'vpm057.front.sepia.ceph.com', 'timeout': 60} 2014-09-25 21:43:45,284.284 INFO:teuthology.task.internal:roles: ubuntu@vpm057.front.sepia.ceph.com - ['mon.a', 'mon.b', 'mon.c', 'osd.0', 'osd.1', 'osd.2', 'osd.3'] 2014-09-25 21:43:45,284.284 INFO:teuthology.run_tasks:Running task internal.push_inventory... 2014-09-25 21:43:45,285.285 INFO:teuthology.orchestra.run.vpm057:Running: 'uname -p' 2014-09-25 21:43:46,325.325 INFO:teuthology.orchestra.run.vpm057:Running: 'cat /etc/os-release' 2014-09-25 21:43:46,960.960 INFO:teuthology.lock:Updating vpm057.front.sepia.ceph.com on lock server 2014-09-25 21:43:46,961.961 INFO:urllib3.connectionpool:Starting new HTTP connection (1): paddles.front.sepia.ceph.com 2014-09-25 21:43:48,012.012 INFO:teuthology.run_tasks:Running task internal.serialize_remote_roles... 2014-09-25 21:43:48,012.012 INFO:teuthology.run_tasks:Running task internal.check_conflict... 2014-09-25 21:43:48,012.012 INFO:teuthology.task.internal:Checking for old test directory... 2014-09-25 21:43:48,013.013 INFO:teuthology.orchestra.run.vpm057:Running: "test '!' -e /home/ubuntu/cephtest" 2014-09-25 21:43:48,480.480 INFO:teuthology.run_tasks:Running task internal.check_ceph_data... 2014-09-25 21:43:48,480.480 INFO:teuthology.task.internal:Checking for old /var/lib/ceph... 2014-09-25 21:43:48,480.480 INFO:teuthology.orchestra.run.vpm057:Running: "test '!' -e /var/lib/ceph" 2014-09-25 21:43:49,111.111 INFO:teuthology.run_tasks:Running task internal.vm_setup... 2014-09-25 21:43:49,111.111 INFO:teuthology.orchestra.run.vpm057:Running: 'test -e /ceph-qa-ready' Connection to vpm057.front.sepia.ceph.com closed. 2014-09-25 21:43:52,796.796 INFO:teuthology.run_tasks:Running task internal.base... 2014-09-25 21:43:52,797.797 INFO:teuthology.task.internal:Creating test directory... 2014-09-25 21:43:52,797.797 INFO:teuthology.orchestra.run.vpm057:Running: 'mkdir -m0755 -- /home/ubuntu/cephtest' 2014-09-25 21:43:53,130.130 INFO:teuthology.run_tasks:Running task internal.archive... 2014-09-25 21:43:53,130.130 INFO:teuthology.task.internal:Creating archive directory... 2014-09-25 21:43:53,130.130 INFO:teuthology.orchestra.run.vpm057:Running: 'install -d -m0755 -- /home/ubuntu/cephtest/archive' 2014-09-25 21:43:53,725.725 INFO:teuthology.run_tasks:Running task internal.coredump... 2014-09-25 21:43:53,725.725 INFO:teuthology.task.internal:Enabling coredump saving... 2014-09-25 21:43:53,726.726 INFO:teuthology.orchestra.run.vpm057:Running: 'install -d -m0755 -- /home/ubuntu/cephtest/archive/coredump && sudo sysctl -w kernel.core_pattern=/home/ubuntu/cephtest/archive/coredump/%t.%p.core' 2014-09-25 21:43:54,405.405 INFO:teuthology.orchestra.run.vpm057.stdout:kernel.core_pattern = /home/ubuntu/cephtest/archive/coredump/%t.%p.core 2014-09-25 21:43:54,405.405 INFO:teuthology.run_tasks:Running task internal.sudo... 2014-09-25 21:43:54,406.406 INFO:teuthology.task.internal:Configuring sudo... 2014-09-25 21:43:54,406.406 INFO:teuthology.orchestra.run.vpm057:Running: "sudo sed -i.orig.teuthology -e 's/^\\([^#]*\\) \\(requiretty\\)/\\1 !\\2/g' -e 's/^\\([^#]*\\) !\\(visiblepw\\)/\\1 \\2/g' /etc/sudoers" 2014-09-25 21:43:54,990.990 INFO:teuthology.run_tasks:Running task internal.syslog... 2014-09-25 21:43:54,991.991 INFO:teuthology.run_tasks:Running task internal.timer... 2014-09-25 21:43:54,991.991 INFO:teuthology.task.internal:Starting timer... 2014-09-25 21:43:54,991.991 INFO:teuthology.run_tasks:Running task install... 2014-09-25 21:43:54,991.991 DEBUG:teuthology.task.install:project ceph 2014-09-25 21:43:54,991.991 DEBUG:teuthology.task.install:config {} 2014-09-25 21:43:54,991.991 INFO:teuthology.task.install:extra packages: [] 2014-09-25 21:43:54,991.991 INFO:teuthology.orchestra.run.vpm057:Running: 'sudo lsb_release -is' 2014-09-25 21:43:55,925.925 DEBUG:teuthology.misc:System to be installed: Ubuntu 2014-09-25 21:43:55,925.925 INFO:teuthology.orchestra.run.vpm057:Running: 'sudo apt-key list | grep Ceph' 2014-09-25 21:43:56,675.675 INFO:teuthology.orchestra.run.vpm057:Running: 'arch' 2014-09-25 21:43:57,187.187 INFO:teuthology.orchestra.run.vpm057:Running: 'lsb_release -is' 2014-09-25 21:43:57,786.786 INFO:teuthology.orchestra.run.vpm057:Running: 'lsb_release -rs' 2014-09-25 21:43:58,367.367 INFO:teuthology.orchestra.run.vpm057:Running: 'lsb_release -sc' 2014-09-25 21:43:59,053.053 INFO:teuthology.task.install:config is {'sha1': None, 'wait_for_package': False, 'project': 'ceph', 'tag': None, 'extra_packages': [], 'branch': None, 'flavor': 'basic', 'extras': None} 2014-09-25 21:43:59,053.053 DEBUG:teuthology.task.install:defaulting to master branch 2014-09-25 21:43:59,053.053 INFO:teuthology.task.install:Installing packages: ceph, ceph-dbg, ceph-mds, ceph-mds-dbg, ceph-common, ceph-common-dbg, ceph-fuse, ceph-fuse-dbg, ceph-test, ceph-test-dbg, radosgw, radosgw-dbg, python-ceph, libcephfs1, libcephfs1-dbg, libcephfs-java, librados2, librados2-dbg, librbd1, librbd1-dbg on remote deb x86_64 2014-09-25 21:43:59,054.054 INFO:teuthology.orchestra.run.vpm057:Running: 'arch' 2014-09-25 21:44:00,536.536 INFO:teuthology.orchestra.run.vpm057:Running: 'lsb_release -is' 2014-09-25 21:44:01,117.117 INFO:teuthology.orchestra.run.vpm057:Running: 'lsb_release -rs' 2014-09-25 21:44:01,763.763 INFO:teuthology.orchestra.run.vpm057:Running: 'lsb_release -sc' 2014-09-25 21:44:02,313.313 INFO:teuthology.task.install:config is {'sha1': None, 'wait_for_package': False, 'project': 'ceph', 'tag': None, 'extra_packages': [], 'branch': None, 'flavor': 'basic', 'extras': None} 2014-09-25 21:44:02,314.314 DEBUG:teuthology.task.install:defaulting to master branch 2014-09-25 21:44:02,314.314 INFO:teuthology.orchestra.run.vpm057:Running: 'sudo lsb_release -is' 2014-09-25 21:44:02,950.950 DEBUG:teuthology.misc:System to be installed: Ubuntu 2014-09-25 21:44:02,950.950 INFO:teuthology.task.install:Pulling from http://gitbuilder.ceph.com/ceph-deb-trusty-x86_64-basic/ref/master 2014-09-25 21:44:02,950.950 INFO:teuthology.orchestra.run.vpm057:Running: 'wget -q -O- http://gitbuilder.ceph.com/ceph-deb-trusty-x86_64-basic/ref/master/version' 2014-09-25 21:44:03,518.518 INFO:teuthology.task.install:Package version is 0.85-988-g7552571-1trusty 2014-09-25 21:44:03,518.518 INFO:teuthology.orchestra.run.vpm057:Running: 'echo deb http://gitbuilder.ceph.com/ceph-deb-trusty-x86_64-basic/ref/master trusty main | sudo tee /etc/apt/sources.list.d/ceph.list' 2014-09-25 21:44:04,160.160 INFO:teuthology.orchestra.run.vpm057:Running: 'sudo apt-get update && sudo DEBIAN_FRONTEND=noninteractive apt-get -y --force-yes -o Dpkg::Options::="--force-confdef" -o Dpkg::Options::="--force-confold" install ceph=0.85-988-g7552571-1trusty ceph-dbg=0.85-988-g7552571-1trusty ceph-mds=0.85-988-g7552571-1trusty ceph-mds-dbg=0.85-988-g7552571-1trusty ceph-common=0.85-988-g7552571-1trusty ceph-common-dbg=0.85-988-g7552571-1trusty ceph-fuse=0.85-988-g7552571-1trusty ceph-fuse-dbg=0.85-988-g7552571-1trusty ceph-test=0.85-988-g7552571-1trusty ceph-test-dbg=0.85-988-g7552571-1trusty radosgw=0.85-988-g7552571-1trusty radosgw-dbg=0.85-988-g7552571-1trusty python-ceph=0.85-988-g7552571-1trusty libcephfs1=0.85-988-g7552571-1trusty libcephfs1-dbg=0.85-988-g7552571-1trusty libcephfs-java=0.85-988-g7552571-1trusty librados2=0.85-988-g7552571-1trusty librados2-dbg=0.85-988-g7552571-1trusty librbd1=0.85-988-g7552571-1trusty librbd1-dbg=0.85-988-g7552571-1trusty' Extracting templates from packages: 100% 2014-09-25 21:46:07,713.713 DEBUG:teuthology.parallel:result is None 2014-09-25 21:46:07,714.714 INFO:teuthology.task.install:Shipping valgrind.supp... 2014-09-25 21:46:07,714.714 INFO:teuthology.orchestra.run.vpm057:Running: 'sudo python -c \'import shutil, sys; shutil.copyfileobj(sys.stdin, file(sys.argv[1], "wb"))\' /home/ubuntu/cephtest/valgrind.supp' 2014-09-25 21:46:09,319.319 INFO:teuthology.task.install:Shipping 'daemon-helper'... 2014-09-25 21:46:09,320.320 INFO:teuthology.orchestra.run.vpm057:Running: 'sudo python -c \'import shutil, sys; shutil.copyfileobj(sys.stdin, file(sys.argv[1], "wb"))\' /usr/bin/daemon-helper' 2014-09-25 21:46:10,272.272 INFO:teuthology.orchestra.run.vpm057:Running: 'sudo chmod a=rx -- /usr/bin/daemon-helper' 2014-09-25 21:46:10,820.820 INFO:teuthology.task.install:Shipping 'adjust-ulimits'... 2014-09-25 21:46:10,820.820 INFO:teuthology.orchestra.run.vpm057:Running: 'sudo python -c \'import shutil, sys; shutil.copyfileobj(sys.stdin, file(sys.argv[1], "wb"))\' /usr/bin/adjust-ulimits' 2014-09-25 21:46:11,817.817 INFO:teuthology.orchestra.run.vpm057:Running: 'sudo chmod a=rx -- /usr/bin/adjust-ulimits' 2014-09-25 21:46:12,459.459 INFO:teuthology.run_tasks:Running task ceph... 2014-09-25 21:46:12,466.466 INFO:tasks.ceph:Making ceph log dir writeable by non-root... 2014-09-25 21:46:12,466.466 INFO:teuthology.orchestra.run.vpm057:Running: 'sudo chmod 777 /var/log/ceph' 2014-09-25 21:46:13,061.061 INFO:tasks.ceph:Disabling ceph logrotate... 2014-09-25 21:46:13,062.062 INFO:teuthology.orchestra.run.vpm057:Running: 'sudo rm -f -- /etc/logrotate.d/ceph' 2014-09-25 21:46:13,927.927 INFO:tasks.ceph:Creating extra log directories... 2014-09-25 21:46:13,928.928 INFO:teuthology.orchestra.run.vpm057:Running: 'sudo install -d -m0755 -- /var/log/ceph/valgrind /var/log/ceph/profiling-logger' 2014-09-25 21:46:14,555.555 INFO:tasks.ceph:Creating ceph cluster... 2014-09-25 21:46:14,556.556 INFO:teuthology.orchestra.run.vpm057:Running: 'install -d -m0755 -- /home/ubuntu/cephtest/data' 2014-09-25 21:46:15,207.207 INFO:teuthology.orchestra.run.vpm057:Running: 'sudo install -d -m0777 -- /var/run/ceph' 2014-09-25 21:46:16,758.758 INFO:teuthology.orchestra.run.vpm057:Running: 'ls /dev/[sv]d?' 2014-09-25 21:46:17,174.174 WARNING:teuthology.misc:Removing root device: /dev/vda from device list 2014-09-25 21:46:17,174.174 DEBUG:teuthology.misc:devs=['/dev/vdb', '/dev/vdc', '/dev/vdd'] 2014-09-25 21:46:17,174.174 INFO:teuthology.orchestra.run.vpm057:Running: 'stat /dev/vdb && sudo dd if=/dev/vdb of=/dev/null count=1 && ! mount | grep -q /dev/vdb' 2014-09-25 21:46:17,730.730 INFO:teuthology.orchestra.run.vpm057.stdout: File: ‘/dev/vdb’ 2014-09-25 21:46:17,730.730 INFO:teuthology.orchestra.run.vpm057.stdout: Size: 0 Blocks: 0 IO Block: 4096 block special file 2014-09-25 21:46:17,730.730 INFO:teuthology.orchestra.run.vpm057.stdout:Device: 5h/5d Inode: 5850 Links: 1 Device type: fd,10 2014-09-25 21:46:17,730.730 INFO:teuthology.orchestra.run.vpm057.stdout:Access: (0660/brw-rw----) Uid: ( 0/ root) Gid: ( 6/ disk) 2014-09-25 21:46:17,730.730 INFO:teuthology.orchestra.run.vpm057.stdout:Access: 2014-09-25 18:40:38.295412000 +0000 2014-09-25 21:46:17,730.730 INFO:teuthology.orchestra.run.vpm057.stdout:Modify: 2014-09-25 18:40:36.479412000 +0000 2014-09-25 21:46:17,730.730 INFO:teuthology.orchestra.run.vpm057.stdout:Change: 2014-09-25 18:40:36.479412000 +0000 2014-09-25 21:46:17,730.730 INFO:teuthology.orchestra.run.vpm057.stdout: Birth: - 2014-09-25 21:46:17,741.741 INFO:teuthology.orchestra.run.vpm057.stderr:1+0 records in 2014-09-25 21:46:17,741.741 INFO:teuthology.orchestra.run.vpm057.stderr:1+0 records out 2014-09-25 21:46:17,742.742 INFO:teuthology.orchestra.run.vpm057.stderr:512 bytes (512 B) copied, 0.00073844 s, 693 kB/s 2014-09-25 21:46:17,745.745 INFO:teuthology.orchestra.run.vpm057:Running: 'stat /dev/vdc && sudo dd if=/dev/vdc of=/dev/null count=1 && ! mount | grep -q /dev/vdc' 2014-09-25 21:46:18,346.346 INFO:teuthology.orchestra.run.vpm057.stdout: File: ‘/dev/vdc’ 2014-09-25 21:46:18,347.347 INFO:teuthology.orchestra.run.vpm057.stdout: Size: 0 Blocks: 0 IO Block: 4096 block special file 2014-09-25 21:46:18,347.347 INFO:teuthology.orchestra.run.vpm057.stdout:Device: 5h/5d Inode: 5853 Links: 1 Device type: fd,20 2014-09-25 21:46:18,348.348 INFO:teuthology.orchestra.run.vpm057.stdout:Access: (0660/brw-rw----) Uid: ( 0/ root) Gid: ( 6/ disk) 2014-09-25 21:46:18,348.348 INFO:teuthology.orchestra.run.vpm057.stdout:Access: 2014-09-25 18:40:38.307412000 +0000 2014-09-25 21:46:18,348.348 INFO:teuthology.orchestra.run.vpm057.stdout:Modify: 2014-09-25 18:40:36.479412000 +0000 2014-09-25 21:46:18,348.348 INFO:teuthology.orchestra.run.vpm057.stdout:Change: 2014-09-25 18:40:36.479412000 +0000 2014-09-25 21:46:18,348.348 INFO:teuthology.orchestra.run.vpm057.stdout: Birth: - 2014-09-25 21:46:18,354.354 INFO:teuthology.orchestra.run.vpm057.stderr:1+0 records in 2014-09-25 21:46:18,354.354 INFO:teuthology.orchestra.run.vpm057.stderr:1+0 records out 2014-09-25 21:46:18,355.355 INFO:teuthology.orchestra.run.vpm057.stderr:512 bytes (512 B) copied, 0.00110454 s, 464 kB/s 2014-09-25 21:46:18,357.357 INFO:teuthology.orchestra.run.vpm057:Running: 'stat /dev/vdd && sudo dd if=/dev/vdd of=/dev/null count=1 && ! mount | grep -q /dev/vdd' 2014-09-25 21:46:18,959.959 INFO:teuthology.orchestra.run.vpm057.stdout: File: ‘/dev/vdd’ 2014-09-25 21:46:18,959.959 INFO:teuthology.orchestra.run.vpm057.stdout: Size: 0 Blocks: 0 IO Block: 4096 block special file 2014-09-25 21:46:18,959.959 INFO:teuthology.orchestra.run.vpm057.stdout:Device: 5h/5d Inode: 5856 Links: 1 Device type: fd,30 2014-09-25 21:46:18,959.959 INFO:teuthology.orchestra.run.vpm057.stdout:Access: (0660/brw-rw----) Uid: ( 0/ root) Gid: ( 6/ disk) 2014-09-25 21:46:18,960.960 INFO:teuthology.orchestra.run.vpm057.stdout:Access: 2014-09-25 18:40:38.327412000 +0000 2014-09-25 21:46:18,960.960 INFO:teuthology.orchestra.run.vpm057.stdout:Modify: 2014-09-25 18:40:36.483412000 +0000 2014-09-25 21:46:18,960.960 INFO:teuthology.orchestra.run.vpm057.stdout:Change: 2014-09-25 18:40:36.483412000 +0000 2014-09-25 21:46:18,960.960 INFO:teuthology.orchestra.run.vpm057.stdout: Birth: - 2014-09-25 21:46:18,965.965 INFO:teuthology.orchestra.run.vpm057.stderr:1+0 records in 2014-09-25 21:46:18,965.965 INFO:teuthology.orchestra.run.vpm057.stderr:1+0 records out 2014-09-25 21:46:18,966.966 INFO:teuthology.orchestra.run.vpm057.stderr:512 bytes (512 B) copied, 0.000717837 s, 713 kB/s 2014-09-25 21:46:18,969.969 INFO:tasks.ceph:dev map: {} 2014-09-25 21:46:18,969.969 INFO:tasks.ceph:Generating config... 2014-09-25 21:46:18,969.969 INFO:tasks.ceph:Setting up mon.a... 2014-09-25 21:46:18,970.970 INFO:teuthology.orchestra.run.vpm057:Running: 'sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph-authtool --create-keyring /etc/ceph/ceph.keyring' 2014-09-25 21:46:19,676.676 INFO:teuthology.orchestra.run.vpm057.stdout:creating /etc/ceph/ceph.keyring 2014-09-25 21:46:19,679.679 INFO:teuthology.orchestra.run.vpm057:Running: 'sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph-authtool --gen-key --name=mon. /etc/ceph/ceph.keyring' 2014-09-25 21:46:20,241.241 INFO:teuthology.orchestra.run.vpm057:Running: 'sudo chmod 0644 /etc/ceph/ceph.keyring' 2014-09-25 21:46:20,886.886 DEBUG:teuthology.misc:Ceph mon addresses: [('a', '10.214.138.97:6789'), ('c', '10.214.138.97:6791'), ('b', '10.214.138.97:6790')] 2014-09-25 21:46:20,887.887 INFO:teuthology.orchestra.run.vpm057:Running: 'adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage monmaptool --create --clobber --add a 10.214.138.97:6789 --add c 10.214.138.97:6791 --add b 10.214.138.97:6790 --print /home/ubuntu/cephtest/monmap' 2014-09-25 21:46:21,533.533 INFO:tasks.ceph:Writing ceph.conf for FSID a0af5d53-05bd-40c8-9d30-7782574e4764... 2014-09-25 21:46:21,535.535 INFO:teuthology.orchestra.run.vpm057:Running: 'sudo mkdir -p /etc/ceph && sudo chmod 0755 /etc/ceph && sudo python -c \'import shutil, sys; shutil.copyfileobj(sys.stdin, file(sys.argv[1], "wb"))\' /etc/ceph/ceph.conf && sudo chmod 0644 /etc/ceph/ceph.conf' 2014-09-25 21:46:22,030.030 WARNING:tasks.ceph:writes: 2014-09-25 21:46:22,376.376 INFO:tasks.ceph:Creating admin key on mon.a... 2014-09-25 21:46:22,377.377 INFO:teuthology.orchestra.run.vpm057:Running: "sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph-authtool --gen-key --name=client.admin --set-uid=0 --cap mon 'allow *' --cap osd 'allow *' --cap mds allow /etc/ceph/ceph.keyring" 2014-09-25 21:46:22,955.955 INFO:tasks.ceph:Copying monmap to all nodes... 2014-09-25 21:46:27,045.045 INFO:tasks.ceph:Sending monmap to node ubuntu@vpm057.front.sepia.ceph.com 2014-09-25 21:46:27,045.045 INFO:teuthology.orchestra.run.vpm057:Running: 'sudo python -c \'import shutil, sys; shutil.copyfileobj(sys.stdin, file(sys.argv[1], "wb"))\' /etc/ceph/ceph.keyring && sudo chmod 0644 /etc/ceph/ceph.keyring' 2014-09-25 21:46:27,785.785 INFO:teuthology.orchestra.run.vpm057:Running: 'python -c \'import shutil, sys; shutil.copyfileobj(sys.stdin, file(sys.argv[1], "wb"))\' /home/ubuntu/cephtest/monmap' 2014-09-25 21:46:28,801.801 INFO:tasks.ceph:Setting up mon nodes... 2014-09-25 21:46:28,801.801 INFO:teuthology.orchestra.run.vpm057:Running: 'adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage osdmaptool -c /etc/ceph/ceph.conf --clobber --createsimple 4 /home/ubuntu/cephtest/osdmap --pg_bits 2 --pgp_bits 4' 2014-09-25 21:46:29,497.497 INFO:teuthology.orchestra.run.vpm057.stderr:osdmaptool: osdmap file '/home/ubuntu/cephtest/osdmap' 2014-09-25 21:46:29,499.499 INFO:teuthology.orchestra.run.vpm057.stdout:osdmaptool: writing epoch 1 to /home/ubuntu/cephtest/osdmap 2014-09-25 21:46:29,501.501 INFO:tasks.ceph:Setting up mds nodes... 2014-09-25 21:46:29,502.502 INFO:tasks.ceph_client:Setting up client nodes... 2014-09-25 21:46:29,502.502 INFO:tasks.ceph:Running mkfs on osd nodes... 2014-09-25 21:46:29,502.502 INFO:tasks.ceph:ctx.disk_config.remote_to_roles_to_dev: {Remote(name='ubuntu@vpm057.front.sepia.ceph.com'): {}} 2014-09-25 21:46:29,503.503 INFO:teuthology.orchestra.run.vpm057:Running: 'sudo mkdir -p /var/lib/ceph/osd/ceph-0' 2014-09-25 21:46:29,932.932 INFO:tasks.ceph:{} 2014-09-25 21:46:29,932.932 INFO:tasks.ceph:0 2014-09-25 21:46:29,933.933 INFO:teuthology.orchestra.run.vpm057:Running: 'sudo mkdir -p /var/lib/ceph/osd/ceph-1' 2014-09-25 21:46:30,475.475 INFO:tasks.ceph:{} 2014-09-25 21:46:30,475.475 INFO:tasks.ceph:1 2014-09-25 21:46:30,475.475 INFO:teuthology.orchestra.run.vpm057:Running: 'sudo mkdir -p /var/lib/ceph/osd/ceph-2' 2014-09-25 21:46:31,033.033 INFO:tasks.ceph:{} 2014-09-25 21:46:31,033.033 INFO:tasks.ceph:2 2014-09-25 21:46:31,033.033 INFO:teuthology.orchestra.run.vpm057:Running: 'sudo mkdir -p /var/lib/ceph/osd/ceph-3' 2014-09-25 21:46:31,824.824 INFO:tasks.ceph:{} 2014-09-25 21:46:31,824.824 INFO:tasks.ceph:3 2014-09-25 21:46:31,824.824 INFO:teuthology.orchestra.run.vpm057:Running: 'sudo MALLOC_CHECK_=3 adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph-osd --mkfs --mkkey -i 0 --monmap /home/ubuntu/cephtest/monmap' 2014-09-25 21:46:32,563.563 INFO:teuthology.orchestra.run.vpm057.stderr:2014-09-25 19:46:32.479055 7ff8e7da7900 -1 journal FileJournal::_open: disabling aio for non-block journal. Use journal_force_aio to force use of aio anyway 2014-09-25 21:46:32,608.608 INFO:teuthology.orchestra.run.vpm057.stderr:2014-09-25 19:46:32.524043 7ff8e7da7900 -1 journal FileJournal::_open: disabling aio for non-block journal. Use journal_force_aio to force use of aio anyway 2014-09-25 21:46:32,819.819 INFO:teuthology.orchestra.run.vpm057.stderr:2014-09-25 19:46:32.525477 7ff8e7da7900 -1 filestore(/var/lib/ceph/osd/ceph-0) could not find 23c2fcde/osd_superblock/0//-1 in index: (2) No such file or directory 2014-09-25 21:46:32,819.819 INFO:teuthology.orchestra.run.vpm057.stderr:2014-09-25 19:46:32.554637 7ff8e7da7900 -1 created object store /var/lib/ceph/osd/ceph-0 journal /var/lib/ceph/osd/ceph-0/journal for osd.0 fsid a0af5d53-05bd-40c8-9d30-7782574e4764 2014-09-25 21:46:32,819.819 INFO:teuthology.orchestra.run.vpm057.stderr:2014-09-25 19:46:32.554971 7ff8e7da7900 -1 auth: error reading file: /var/lib/ceph/osd/ceph-0/keyring: can't open /var/lib/ceph/osd/ceph-0/keyring: (2) No such file or directory 2014-09-25 21:46:32,819.819 INFO:teuthology.orchestra.run.vpm057.stderr:2014-09-25 19:46:32.555322 7ff8e7da7900 -1 created new key in keyring /var/lib/ceph/osd/ceph-0/keyring 2014-09-25 21:46:32,820.820 INFO:teuthology.orchestra.run.vpm057:Running: 'sudo MALLOC_CHECK_=3 adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph-osd --mkfs --mkkey -i 1 --monmap /home/ubuntu/cephtest/monmap' 2014-09-25 21:46:33,454.454 INFO:teuthology.orchestra.run.vpm057.stderr:2014-09-25 19:46:33.369730 7fcef6685900 -1 journal FileJournal::_open: disabling aio for non-block journal. Use journal_force_aio to force use of aio anyway 2014-09-25 21:46:33,677.677 INFO:teuthology.orchestra.run.vpm057.stderr:2014-09-25 19:46:33.591981 7fcef6685900 -1 journal FileJournal::_open: disabling aio for non-block journal. Use journal_force_aio to force use of aio anyway 2014-09-25 21:46:33,679.679 INFO:teuthology.orchestra.run.vpm057.stderr:2014-09-25 19:46:33.593064 7fcef6685900 -1 filestore(/var/lib/ceph/osd/ceph-1) could not find 23c2fcde/osd_superblock/0//-1 in index: (2) No such file or directory 2014-09-25 21:46:33,853.853 INFO:teuthology.orchestra.run.vpm057.stderr:2014-09-25 19:46:33.620232 7fcef6685900 -1 created object store /var/lib/ceph/osd/ceph-1 journal /var/lib/ceph/osd/ceph-1/journal for osd.1 fsid a0af5d53-05bd-40c8-9d30-7782574e4764 2014-09-25 21:46:33,853.853 INFO:teuthology.orchestra.run.vpm057.stderr:2014-09-25 19:46:33.620611 7fcef6685900 -1 auth: error reading file: /var/lib/ceph/osd/ceph-1/keyring: can't open /var/lib/ceph/osd/ceph-1/keyring: (2) No such file or directory 2014-09-25 21:46:33,853.853 INFO:teuthology.orchestra.run.vpm057.stderr:2014-09-25 19:46:33.620966 7fcef6685900 -1 created new key in keyring /var/lib/ceph/osd/ceph-1/keyring 2014-09-25 21:46:33,853.853 INFO:teuthology.orchestra.run.vpm057:Running: 'sudo MALLOC_CHECK_=3 adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph-osd --mkfs --mkkey -i 2 --monmap /home/ubuntu/cephtest/monmap' 2014-09-25 21:46:34,458.458 INFO:teuthology.orchestra.run.vpm057.stderr:2014-09-25 19:46:34.367641 7fc1731bb900 -1 journal FileJournal::_open: disabling aio for non-block journal. Use journal_force_aio to force use of aio anyway 2014-09-25 21:46:34,492.492 INFO:teuthology.orchestra.run.vpm057.stderr:2014-09-25 19:46:34.407603 7fc1731bb900 -1 journal FileJournal::_open: disabling aio for non-block journal. Use journal_force_aio to force use of aio anyway 2014-09-25 21:46:34,494.494 INFO:teuthology.orchestra.run.vpm057.stderr:2014-09-25 19:46:34.408766 7fc1731bb900 -1 filestore(/var/lib/ceph/osd/ceph-2) could not find 23c2fcde/osd_superblock/0//-1 in index: (2) No such file or directory 2014-09-25 21:46:34,566.566 INFO:teuthology.orchestra.run.vpm057.stderr:2014-09-25 19:46:34.431359 7fc1731bb900 -1 created object store /var/lib/ceph/osd/ceph-2 journal /var/lib/ceph/osd/ceph-2/journal for osd.2 fsid a0af5d53-05bd-40c8-9d30-7782574e4764 2014-09-25 21:46:34,566.566 INFO:teuthology.orchestra.run.vpm057.stderr:2014-09-25 19:46:34.431676 7fc1731bb900 -1 auth: error reading file: /var/lib/ceph/osd/ceph-2/keyring: can't open /var/lib/ceph/osd/ceph-2/keyring: (2) No such file or directory 2014-09-25 21:46:34,566.566 INFO:teuthology.orchestra.run.vpm057.stderr:2014-09-25 19:46:34.432052 7fc1731bb900 -1 created new key in keyring /var/lib/ceph/osd/ceph-2/keyring 2014-09-25 21:46:34,566.566 INFO:teuthology.orchestra.run.vpm057:Running: 'sudo MALLOC_CHECK_=3 adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph-osd --mkfs --mkkey -i 3 --monmap /home/ubuntu/cephtest/monmap' 2014-09-25 21:46:35,138.138 INFO:teuthology.orchestra.run.vpm057.stderr:2014-09-25 19:46:35.054445 7fc185f9d900 -1 journal FileJournal::_open: disabling aio for non-block journal. Use journal_force_aio to force use of aio anyway 2014-09-25 21:46:35,335.335 INFO:teuthology.orchestra.run.vpm057.stderr:2014-09-25 19:46:35.089517 7fc185f9d900 -1 journal FileJournal::_open: disabling aio for non-block journal. Use journal_force_aio to force use of aio anyway 2014-09-25 21:46:35,335.335 INFO:teuthology.orchestra.run.vpm057.stderr:2014-09-25 19:46:35.090614 7fc185f9d900 -1 filestore(/var/lib/ceph/osd/ceph-3) could not find 23c2fcde/osd_superblock/0//-1 in index: (2) No such file or directory 2014-09-25 21:46:35,335.335 INFO:teuthology.orchestra.run.vpm057.stderr:2014-09-25 19:46:35.113615 7fc185f9d900 -1 created object store /var/lib/ceph/osd/ceph-3 journal /var/lib/ceph/osd/ceph-3/journal for osd.3 fsid a0af5d53-05bd-40c8-9d30-7782574e4764 2014-09-25 21:46:35,335.335 INFO:teuthology.orchestra.run.vpm057.stderr:2014-09-25 19:46:35.113940 7fc185f9d900 -1 auth: error reading file: /var/lib/ceph/osd/ceph-3/keyring: can't open /var/lib/ceph/osd/ceph-3/keyring: (2) No such file or directory 2014-09-25 21:46:35,335.335 INFO:teuthology.orchestra.run.vpm057.stderr:2014-09-25 19:46:35.114294 7fc185f9d900 -1 created new key in keyring /var/lib/ceph/osd/ceph-3/keyring 2014-09-25 21:46:35,335.335 INFO:tasks.ceph:Reading keys from all nodes... 2014-09-25 21:46:35,336.336 INFO:teuthology.orchestra.run.vpm057:Running: "python -c 'import os; import tempfile; import sys;(fd,fname) = tempfile.mkstemp();os.close(fd);sys.stdout.write(fname.rstrip());sys.stdout.flush()'" 2014-09-25 21:46:35,892.892 INFO:teuthology.orchestra.run.vpm057:Running: 'sudo cp /var/lib/ceph/osd/ceph-0/keyring /tmp/tmpARj8Oj' 2014-09-25 21:46:36,405.405 INFO:teuthology.orchestra.run.vpm057:Running: 'sudo chmod 0666 /tmp/tmpARj8Oj' 2014-09-25 21:46:39,211.211 INFO:teuthology.orchestra.run.vpm057:Running: 'rm -fr /tmp/tmpARj8Oj' 2014-09-25 21:46:39,562.562 INFO:teuthology.orchestra.run.vpm057:Running: "python -c 'import os; import tempfile; import sys;(fd,fname) = tempfile.mkstemp();os.close(fd);sys.stdout.write(fname.rstrip());sys.stdout.flush()'" 2014-09-25 21:46:40,166.166 INFO:teuthology.orchestra.run.vpm057:Running: 'sudo cp /var/lib/ceph/osd/ceph-1/keyring /tmp/tmpC_GPbX' 2014-09-25 21:46:40,782.782 INFO:teuthology.orchestra.run.vpm057:Running: 'sudo chmod 0666 /tmp/tmpC_GPbX' 2014-09-25 21:46:43,497.497 INFO:teuthology.orchestra.run.vpm057:Running: 'rm -fr /tmp/tmpC_GPbX' 2014-09-25 21:46:44,038.038 INFO:teuthology.orchestra.run.vpm057:Running: "python -c 'import os; import tempfile; import sys;(fd,fname) = tempfile.mkstemp();os.close(fd);sys.stdout.write(fname.rstrip());sys.stdout.flush()'" 2014-09-25 21:46:44,802.802 INFO:teuthology.orchestra.run.vpm057:Running: 'sudo cp /var/lib/ceph/osd/ceph-2/keyring /tmp/tmp4Ahv_D' 2014-09-25 21:46:45,621.621 INFO:teuthology.orchestra.run.vpm057:Running: 'sudo chmod 0666 /tmp/tmp4Ahv_D' 2014-09-25 21:46:48,158.158 INFO:teuthology.orchestra.run.vpm057:Running: 'rm -fr /tmp/tmp4Ahv_D' 2014-09-25 21:46:48,490.490 INFO:teuthology.orchestra.run.vpm057:Running: "python -c 'import os; import tempfile; import sys;(fd,fname) = tempfile.mkstemp();os.close(fd);sys.stdout.write(fname.rstrip());sys.stdout.flush()'" 2014-09-25 21:46:48,997.997 INFO:teuthology.orchestra.run.vpm057:Running: 'sudo cp /var/lib/ceph/osd/ceph-3/keyring /tmp/tmpP6LxSW' 2014-09-25 21:46:49,615.615 INFO:teuthology.orchestra.run.vpm057:Running: 'sudo chmod 0666 /tmp/tmpP6LxSW' 2014-09-25 21:46:52,303.303 INFO:teuthology.orchestra.run.vpm057:Running: 'rm -fr /tmp/tmpP6LxSW' 2014-09-25 21:46:52,771.771 INFO:tasks.ceph:Adding keys to all mons... 2014-09-25 21:46:52,771.771 INFO:teuthology.orchestra.run.vpm057:Running: 'sudo tee -a /etc/ceph/ceph.keyring' 2014-09-25 21:46:53,664.664 INFO:teuthology.orchestra.run.vpm057:Running: "sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph-authtool /etc/ceph/ceph.keyring --name=osd.0 --cap osd 'allow *' --cap mon 'allow *'" 2014-09-25 21:46:54,428.428 INFO:teuthology.orchestra.run.vpm057:Running: "sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph-authtool /etc/ceph/ceph.keyring --name=osd.1 --cap osd 'allow *' --cap mon 'allow *'" 2014-09-25 21:46:55,251.251 INFO:teuthology.orchestra.run.vpm057:Running: "sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph-authtool /etc/ceph/ceph.keyring --name=osd.2 --cap osd 'allow *' --cap mon 'allow *'" 2014-09-25 21:46:56,048.048 INFO:teuthology.orchestra.run.vpm057:Running: "sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph-authtool /etc/ceph/ceph.keyring --name=osd.3 --cap osd 'allow *' --cap mon 'allow *'" 2014-09-25 21:46:56,869.869 INFO:tasks.ceph:Running mkfs on mon nodes... 2014-09-25 21:46:56,870.870 INFO:teuthology.orchestra.run.vpm057:Running: 'sudo mkdir -p /var/lib/ceph/mon/ceph-a' 2014-09-25 21:46:57,510.510 INFO:teuthology.orchestra.run.vpm057:Running: 'sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph-mon --mkfs -i a --monmap=/home/ubuntu/cephtest/monmap --osdmap=/home/ubuntu/cephtest/osdmap --keyring=/etc/ceph/ceph.keyring' 2014-09-25 21:46:58,289.289 INFO:teuthology.orchestra.run.vpm057.stdout:ceph-mon: set fsid to a0af5d53-05bd-40c8-9d30-7782574e4764 2014-09-25 21:46:58,306.306 INFO:teuthology.orchestra.run.vpm057.stdout:ceph-mon: created monfs at /var/lib/ceph/mon/ceph-a for mon.a 2014-09-25 21:46:58,311.311 INFO:teuthology.orchestra.run.vpm057:Running: 'sudo mkdir -p /var/lib/ceph/mon/ceph-b' 2014-09-25 21:46:59,052.052 INFO:teuthology.orchestra.run.vpm057:Running: 'sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph-mon --mkfs -i b --monmap=/home/ubuntu/cephtest/monmap --osdmap=/home/ubuntu/cephtest/osdmap --keyring=/etc/ceph/ceph.keyring' 2014-09-25 21:46:59,611.611 INFO:teuthology.orchestra.run.vpm057.stdout:ceph-mon: set fsid to a0af5d53-05bd-40c8-9d30-7782574e4764 2014-09-25 21:46:59,628.628 INFO:teuthology.orchestra.run.vpm057.stdout:ceph-mon: created monfs at /var/lib/ceph/mon/ceph-b for mon.b 2014-09-25 21:46:59,632.632 INFO:teuthology.orchestra.run.vpm057:Running: 'sudo mkdir -p /var/lib/ceph/mon/ceph-c' 2014-09-25 21:47:00,368.368 INFO:teuthology.orchestra.run.vpm057:Running: 'sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph-mon --mkfs -i c --monmap=/home/ubuntu/cephtest/monmap --osdmap=/home/ubuntu/cephtest/osdmap --keyring=/etc/ceph/ceph.keyring' 2014-09-25 21:47:01,289.289 INFO:teuthology.orchestra.run.vpm057.stdout:ceph-mon: set fsid to a0af5d53-05bd-40c8-9d30-7782574e4764 2014-09-25 21:47:01,289.289 INFO:teuthology.orchestra.run.vpm057.stdout:ceph-mon: created monfs at /var/lib/ceph/mon/ceph-c for mon.c 2014-09-25 21:47:01,289.289 INFO:teuthology.orchestra.run.vpm057:Running: 'rm -- /home/ubuntu/cephtest/monmap /home/ubuntu/cephtest/osdmap' 2014-09-25 21:47:01,876.876 INFO:tasks.ceph:Starting mon daemons... 2014-09-25 21:47:01,877.877 INFO:tasks.ceph.mon.a:Restarting daemon 2014-09-25 21:47:01,877.877 INFO:teuthology.orchestra.run.vpm057:Running: 'sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage daemon-helper kill ceph-mon -f -i a' 2014-09-25 21:47:02,515.515 INFO:tasks.ceph.mon.a:Started 2014-09-25 21:47:02,516.516 INFO:tasks.ceph.mon.b:Restarting daemon 2014-09-25 21:47:02,516.516 INFO:teuthology.orchestra.run.vpm057:Running: 'sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage daemon-helper kill ceph-mon -f -i b' 2014-09-25 21:47:02,926.926 INFO:tasks.ceph.mon.b:Started 2014-09-25 21:47:02,926.926 INFO:tasks.ceph.mon.c:Restarting daemon 2014-09-25 21:47:02,926.926 INFO:teuthology.orchestra.run.vpm057:Running: 'sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage daemon-helper kill ceph-mon -f -i c' 2014-09-25 21:47:03,254.254 INFO:tasks.ceph.mon.c:Started 2014-09-25 21:47:03,254.254 INFO:tasks.ceph:Starting osd daemons... 2014-09-25 21:47:03,254.254 INFO:tasks.ceph.osd.0:Restarting daemon 2014-09-25 21:47:03,254.254 INFO:teuthology.orchestra.run.vpm057:Running: 'sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage daemon-helper kill ceph-osd -f -i 0' 2014-09-25 21:47:03,583.583 INFO:tasks.ceph.osd.0:Started 2014-09-25 21:47:03,583.583 INFO:tasks.ceph.osd.1:Restarting daemon 2014-09-25 21:47:03,583.583 INFO:teuthology.orchestra.run.vpm057:Running: 'sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage daemon-helper kill ceph-osd -f -i 1' 2014-09-25 21:47:03,626.626 INFO:tasks.ceph.osd.0.vpm057.stdout:starting osd.0 at :/0 osd_data /var/lib/ceph/osd/ceph-0 /var/lib/ceph/osd/ceph-0/journal 2014-09-25 21:47:03,659.659 INFO:tasks.ceph.osd.0.vpm057.stderr:2014-09-25 19:47:03.574871 7f4d90293900 -1 journal FileJournal::_open: disabling aio for non-block journal. Use journal_force_aio to force use of aio anyway 2014-09-25 21:47:04,147.147 INFO:tasks.ceph.osd.1:Started 2014-09-25 21:47:04,147.147 INFO:tasks.ceph.osd.2:Restarting daemon 2014-09-25 21:47:04,147.147 INFO:teuthology.orchestra.run.vpm057:Running: 'sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage daemon-helper kill ceph-osd -f -i 2' 2014-09-25 21:47:04,148.148 INFO:tasks.ceph.osd.1.vpm057.stdout:starting osd.1 at :/0 osd_data /var/lib/ceph/osd/ceph-1 /var/lib/ceph/osd/ceph-1/journal 2014-09-25 21:47:05,781.781 INFO:tasks.ceph.osd.1.vpm057.stderr:2014-09-25 19:47:04.233100 7f1710a6c900 -1 journal FileJournal::_open: disabling aio for non-block journal. Use journal_force_aio to force use of aio anyway 2014-09-25 21:47:07,220.220 INFO:tasks.ceph.osd.2:Started 2014-09-25 21:47:07,220.220 INFO:tasks.ceph.osd.3:Restarting daemon 2014-09-25 21:47:07,220.220 INFO:teuthology.orchestra.run.vpm057:Running: 'sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage daemon-helper kill ceph-osd -f -i 3' 2014-09-25 21:47:07,221.221 INFO:tasks.ceph.osd.2.vpm057.stderr:2014-09-25 19:47:06.174929 7fed266a4900 -1 journal FileJournal::_open: disabling aio for non-block journal. Use journal_force_aio to force use of aio anyway 2014-09-25 21:47:07,221.221 INFO:tasks.ceph.osd.2.vpm057.stdout:starting osd.2 at :/0 osd_data /var/lib/ceph/osd/ceph-2 /var/lib/ceph/osd/ceph-2/journal 2014-09-25 21:47:09,251.251 INFO:tasks.ceph.osd.3:Started 2014-09-25 21:47:09,251.251 INFO:tasks.ceph:Starting mds daemons... 2014-09-25 21:47:09,251.251 INFO:tasks.ceph:Waiting until ceph is healthy... 2014-09-25 21:47:09,252.252 INFO:teuthology.orchestra.run.vpm057:Running: 'adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph osd dump --format=json' 2014-09-25 21:47:09,253.253 INFO:tasks.ceph.osd.3.vpm057.stderr:2014-09-25 19:47:07.577679 7f7209873900 -1 journal FileJournal::_open: disabling aio for non-block journal. Use journal_force_aio to force use of aio anyway 2014-09-25 21:47:09,253.253 INFO:tasks.ceph.osd.3.vpm057.stdout:starting osd.3 at :/0 osd_data /var/lib/ceph/osd/ceph-3 /var/lib/ceph/osd/ceph-3/journal 2014-09-25 21:47:10,815.815 DEBUG:teuthology.misc:2 of 4 OSDs are up 2014-09-25 21:47:11,814.814 INFO:teuthology.orchestra.run.vpm057:Running: 'adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph osd dump --format=json' 2014-09-25 21:47:12,332.332 DEBUG:teuthology.misc:4 of 4 OSDs are up 2014-09-25 21:47:12,332.332 INFO:teuthology.orchestra.run.vpm057:Running: 'adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph health' 2014-09-25 21:47:13,386.386 DEBUG:teuthology.misc:Ceph health: HEALTH_WARN 11 pgs peering; 15 pgs stuck inactive; 15 pgs stuck unclean 2014-09-25 21:47:20,387.387 INFO:teuthology.orchestra.run.vpm057:Running: 'adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph health' 2014-09-25 21:47:21,200.200 DEBUG:teuthology.misc:Ceph health: HEALTH_OK 2014-09-25 21:47:21,200.200 INFO:teuthology.orchestra.run.vpm057:Running: 'adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph osd dump --format=json' 2014-09-25 21:47:21,918.918 INFO:tasks.ceph.ceph_manager:[{u'cache_target_full_ratio_micro': 0, u'stripe_width': 0, u'flags_names': u'hashpspool', u'tier_of': -1, u'pg_placement_num': 16, u'quota_max_bytes': 0, u'erasure_code_profile': u'', u'expected_num_objects': 0, u'size': 2, u'snap_seq': 0, u'auid': 0, u'cache_min_flush_age': 0, u'hit_set_period': 0, u'min_read_recency_for_promote': 0, u'target_max_objects': 0, u'pg_num': 16, u'type': 1, u'crush_ruleset': 0, u'pool_name': u'rbd', u'cache_min_evict_age': 0, u'snap_mode': u'selfmanaged', u'cache_mode': u'none', u'min_size': 1, u'crash_replay_interval': 0, u'object_hash': 2, u'write_tier': -1, u'cache_target_dirty_ratio_micro': 0, u'pool': 0, u'removed_snaps': u'[]', u'tiers': [], u'hit_set_params': {u'type': u'none'}, u'last_force_op_resend': u'0', u'pool_snaps': [], u'quota_max_objects': 0, u'hit_set_count': 0, u'flags': 1, u'target_max_bytes': 0, u'snap_epoch': 0, u'last_change': u'1', u'read_tier': -1}] 2014-09-25 21:47:21,918.918 INFO:teuthology.orchestra.run.vpm057:Running: 'adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph osd pool get rbd pg_num' 2014-09-25 21:47:22,474.474 INFO:teuthology.run_tasks:Running task ec_lost_unfound... 2014-09-25 21:47:22,475.475 INFO:teuthology.orchestra.run.vpm057:Running: 'adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph osd dump --format=json' 2014-09-25 21:47:23,307.307 INFO:tasks.ec_lost_unfound.ceph_manager:[{u'cache_target_full_ratio_micro': 0, u'stripe_width': 0, u'flags_names': u'hashpspool', u'tier_of': -1, u'pg_placement_num': 16, u'quota_max_bytes': 0, u'erasure_code_profile': u'', u'expected_num_objects': 0, u'size': 2, u'snap_seq': 0, u'auid': 0, u'cache_min_flush_age': 0, u'hit_set_period': 0, u'min_read_recency_for_promote': 0, u'target_max_objects': 0, u'pg_num': 16, u'type': 1, u'crush_ruleset': 0, u'pool_name': u'rbd', u'cache_min_evict_age': 0, u'snap_mode': u'selfmanaged', u'cache_mode': u'none', u'min_size': 1, u'crash_replay_interval': 0, u'object_hash': 2, u'write_tier': -1, u'cache_target_dirty_ratio_micro': 0, u'pool': 0, u'removed_snaps': u'[]', u'tiers': [], u'hit_set_params': {u'type': u'none'}, u'last_force_op_resend': u'0', u'pool_snaps': [], u'quota_max_objects': 0, u'hit_set_count': 0, u'flags': 1, u'target_max_bytes': 0, u'snap_epoch': 0, u'last_change': u'1', u'read_tier': -1}] 2014-09-25 21:47:23,308.308 INFO:teuthology.orchestra.run.vpm057:Running: 'adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph osd pool get rbd pg_num' 2014-09-25 21:47:24,001.001 INFO:teuthology.orchestra.run.vpm057:Running: 'adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph tell osd.0 flush_pg_stats' 2014-09-25 21:47:24,737.737 INFO:teuthology.orchestra.run.vpm057:Running: 'adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph tell osd.1 flush_pg_stats' 2014-09-25 21:47:25,453.453 INFO:teuthology.orchestra.run.vpm057:Running: 'adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph tell osd.2 flush_pg_stats' 2014-09-25 21:47:26,348.348 INFO:teuthology.orchestra.run.vpm057:Running: 'adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph tell osd.3 flush_pg_stats' 2014-09-25 21:47:27,183.183 INFO:tasks.ec_lost_unfound.ceph_manager:waiting for clean 2014-09-25 21:47:27,183.183 INFO:teuthology.orchestra.run.vpm057:Running: 'adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph pg dump --format=json' 2014-09-25 21:47:28,012.012 INFO:teuthology.orchestra.run.vpm057.stderr:dumped all in format json 2014-09-25 21:47:28,507.507 INFO:teuthology.orchestra.run.vpm057:Running: 'adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph pg dump --format=json' 2014-09-25 21:47:29,363.363 INFO:teuthology.orchestra.run.vpm057.stderr:dumped all in format json 2014-09-25 21:47:29,544.544 INFO:teuthology.orchestra.run.vpm057:Running: 'adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph status --format=json-pretty' 2014-09-25 21:47:30,256.256 INFO:tasks.ec_lost_unfound.ceph_manager:{u'election_epoch': 4, u'quorum': [0, 1, 2], u'mdsmap': {u'max': 0, u'epoch': 1, u'by_rank': [], u'up': 0, u'in': 0}, u'monmap': {u'epoch': 1, u'mons': [{u'name': u'a', u'rank': 0, u'addr': u'10.214.138.97:6789/0'}, {u'name': u'b', u'rank': 1, u'addr': u'10.214.138.97:6790/0'}, {u'name': u'c', u'rank': 2, u'addr': u'10.214.138.97:6791/0'}], u'modified': u'2014-09-25 19:46:21.446598', u'fsid': u'a0af5d53-05bd-40c8-9d30-7782574e4764', u'created': u'2014-09-25 19:46:21.446598'}, u'health': {u'detail': [], u'timechecks': {u'round_status': u'finished', u'epoch': 4, u'round': 2, u'mons': [{u'latency': u'0.000000', u'skew': u'0.000000', u'health': u'HEALTH_OK', u'name': u'a'}, {u'latency': u'0.003394', u'skew': u'0.000000', u'health': u'HEALTH_OK', u'name': u'b'}, {u'latency': u'0.018347', u'skew': u'0.000000', u'health': u'HEALTH_OK', u'name': u'c'}]}, u'health': {u'health_services': [{u'mons': []}]}, u'overall_status': u'HEALTH_OK', u'summary': []}, u'pgmap': {u'bytes_total': 422619250688, u'num_pgs': 16, u'data_bytes': 0, u'bytes_used': 20672110592, u'version': 11, u'pgs_by_state': [{u'count': 16, u'state_name': u'active+clean'}], u'bytes_avail': 384641703936}, u'quorum_names': [u'a', u'b', u'c'], u'osdmap': {u'osdmap': {u'full': False, u'nearfull': False, u'num_osds': 4, u'num_up_osds': 4, u'epoch': 6, u'num_in_osds': 4}}, u'fsid': u'a0af5d53-05bd-40c8-9d30-7782574e4764'} 2014-09-25 21:47:30,256.256 INFO:tasks.ec_lost_unfound.ceph_manager:clean! 2014-09-25 21:47:30,256.256 INFO:teuthology.orchestra.run.vpm057:Running: 'adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph osd erasure-code-profile set lost_unfound k=2 m=2 ruleset-failure-domain=osd' 2014-09-25 21:47:31,598.598 INFO:tasks.ec_lost_unfound.ceph_manager:creating pool_name unique_pool_0 2014-09-25 21:47:31,599.599 INFO:teuthology.orchestra.run.vpm057:Running: 'adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph osd pool create unique_pool_0 16 16 erasure lost_unfound' 2014-09-25 21:47:33,644.644 INFO:teuthology.orchestra.run.vpm057.stderr:pool 'unique_pool_0' created 2014-09-25 21:47:33,645.645 INFO:tasks.util.rados:rados -p unique_pool_0 put dummy /etc/fstab 2014-09-25 21:47:33,645.645 INFO:teuthology.orchestra.run.vpm057:Running: 'adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage rados -p unique_pool_0 put dummy /etc/fstab' 2014-09-25 21:47:34,868.868 INFO:teuthology.orchestra.run.vpm057:Running: 'adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph tell osd.0 flush_pg_stats' 2014-09-25 21:47:35,796.796 INFO:teuthology.orchestra.run.vpm057:Running: 'adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph tell osd.1 flush_pg_stats' 2014-09-25 21:47:36,740.740 INFO:tasks.ec_lost_unfound.ceph_manager:waiting for recovery to complete 2014-09-25 21:47:36,740.740 INFO:teuthology.orchestra.run.vpm057:Running: 'adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph pg dump --format=json' 2014-09-25 21:47:37,392.392 INFO:teuthology.orchestra.run.vpm057.stderr:dumped all in format json 2014-09-25 21:47:38,080.080 INFO:teuthology.orchestra.run.vpm057:Running: 'adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph pg dump --format=json' 2014-09-25 21:47:39,046.046 INFO:teuthology.orchestra.run.vpm057.stderr:dumped all in format json 2014-09-25 21:47:39,292.292 INFO:teuthology.orchestra.run.vpm057:Running: 'adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph status --format=json-pretty' 2014-09-25 21:47:40,146.146 INFO:tasks.ec_lost_unfound.ceph_manager:{u'election_epoch': 4, u'quorum': [0, 1, 2], u'mdsmap': {u'max': 0, u'epoch': 1, u'by_rank': [], u'up': 0, u'in': 0}, u'monmap': {u'epoch': 1, u'mons': [{u'name': u'a', u'rank': 0, u'addr': u'10.214.138.97:6789/0'}, {u'name': u'b', u'rank': 1, u'addr': u'10.214.138.97:6790/0'}, {u'name': u'c', u'rank': 2, u'addr': u'10.214.138.97:6791/0'}], u'modified': u'2014-09-25 19:46:21.446598', u'fsid': u'a0af5d53-05bd-40c8-9d30-7782574e4764', u'created': u'2014-09-25 19:46:21.446598'}, u'health': {u'detail': [], u'timechecks': {u'round_status': u'finished', u'epoch': 4, u'round': 2, u'mons': [{u'latency': u'0.000000', u'skew': u'0.000000', u'health': u'HEALTH_OK', u'name': u'a'}, {u'latency': u'0.003394', u'skew': u'0.000000', u'health': u'HEALTH_OK', u'name': u'b'}, {u'latency': u'0.018347', u'skew': u'0.000000', u'health': u'HEALTH_OK', u'name': u'c'}]}, u'health': {u'health_services': [{u'mons': []}]}, u'overall_status': u'HEALTH_OK', u'summary': []}, u'pgmap': {u'bytes_total': 422619250688, u'num_pgs': 32, u'data_bytes': 43, u'bytes_used': 20685651968, u'op_per_sec': 0, u'version': 20, u'pgs_by_state': [{u'count': 32, u'state_name': u'active+clean'}], u'write_bytes_sec': 325, u'bytes_avail': 384628162560}, u'quorum_names': [u'a', u'b', u'c'], u'osdmap': {u'osdmap': {u'full': False, u'nearfull': False, u'num_osds': 4, u'num_up_osds': 4, u'epoch': 10, u'num_in_osds': 4}}, u'fsid': u'a0af5d53-05bd-40c8-9d30-7782574e4764'} 2014-09-25 21:47:40,147.147 INFO:teuthology.orchestra.run.vpm057:Running: 'adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph pg dump --format=json' 2014-09-25 21:47:40,915.915 INFO:teuthology.orchestra.run.vpm057.stderr:dumped all in format json 2014-09-25 21:47:44,100.100 INFO:teuthology.orchestra.run.vpm057:Running: 'adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph pg dump --format=json' 2014-09-25 21:47:44,606.606 INFO:teuthology.orchestra.run.vpm057.stderr:dumped all in format json 2014-09-25 21:47:45,090.090 INFO:teuthology.orchestra.run.vpm057:Running: 'adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph status --format=json-pretty' 2014-09-25 21:47:45,730.730 INFO:tasks.ec_lost_unfound.ceph_manager:{u'election_epoch': 4, u'quorum': [0, 1, 2], u'mdsmap': {u'max': 0, u'epoch': 1, u'by_rank': [], u'up': 0, u'in': 0}, u'monmap': {u'epoch': 1, u'mons': [{u'name': u'a', u'rank': 0, u'addr': u'10.214.138.97:6789/0'}, {u'name': u'b', u'rank': 1, u'addr': u'10.214.138.97:6790/0'}, {u'name': u'c', u'rank': 2, u'addr': u'10.214.138.97:6791/0'}], u'modified': u'2014-09-25 19:46:21.446598', u'fsid': u'a0af5d53-05bd-40c8-9d30-7782574e4764', u'created': u'2014-09-25 19:46:21.446598'}, u'health': {u'detail': [], u'timechecks': {u'round_status': u'finished', u'epoch': 4, u'round': 2, u'mons': [{u'latency': u'0.000000', u'skew': u'0.000000', u'health': u'HEALTH_OK', u'name': u'a'}, {u'latency': u'0.003394', u'skew': u'0.000000', u'health': u'HEALTH_OK', u'name': u'b'}, {u'latency': u'0.018347', u'skew': u'0.000000', u'health': u'HEALTH_OK', u'name': u'c'}]}, u'health': {u'health_services': [{u'mons': []}]}, u'overall_status': u'HEALTH_OK', u'summary': []}, u'pgmap': {u'bytes_total': 422619250688, u'num_pgs': 32, u'data_bytes': 43, u'bytes_used': 20685651968, u'op_per_sec': 0, u'version': 20, u'pgs_by_state': [{u'count': 32, u'state_name': u'active+clean'}], u'write_bytes_sec': 325, u'bytes_avail': 384628162560}, u'quorum_names': [u'a', u'b', u'c'], u'osdmap': {u'osdmap': {u'full': False, u'nearfull': False, u'num_osds': 4, u'num_up_osds': 4, u'epoch': 10, u'num_in_osds': 4}}, u'fsid': u'a0af5d53-05bd-40c8-9d30-7782574e4764'} 2014-09-25 21:47:45,730.730 INFO:tasks.ec_lost_unfound.ceph_manager:recovered! 2014-09-25 21:47:45,730.730 INFO:tasks.util.rados:rados -p unique_pool_0 put existing_1 /etc/fstab 2014-09-25 21:47:45,731.731 INFO:teuthology.orchestra.run.vpm057:Running: 'adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage rados -p unique_pool_0 put existing_1 /etc/fstab' 2014-09-25 21:47:46,400.400 INFO:tasks.util.rados:rados -p unique_pool_0 put existed_1 /etc/fstab 2014-09-25 21:47:46,400.400 INFO:teuthology.orchestra.run.vpm057:Running: 'adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage rados -p unique_pool_0 put existed_1 /etc/fstab' 2014-09-25 21:47:47,060.060 INFO:tasks.util.rados:rados -p unique_pool_0 rm existed_1 2014-09-25 21:47:47,060.060 INFO:teuthology.orchestra.run.vpm057:Running: 'adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage rados -p unique_pool_0 rm existed_1' 2014-09-25 21:47:47,777.777 INFO:tasks.util.rados:rados -p unique_pool_0 put existing_2 /etc/fstab 2014-09-25 21:47:47,777.777 INFO:teuthology.orchestra.run.vpm057:Running: 'adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage rados -p unique_pool_0 put existing_2 /etc/fstab' 2014-09-25 21:47:48,246.246 INFO:tasks.util.rados:rados -p unique_pool_0 put existed_2 /etc/fstab 2014-09-25 21:47:48,246.246 INFO:teuthology.orchestra.run.vpm057:Running: 'adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage rados -p unique_pool_0 put existed_2 /etc/fstab' 2014-09-25 21:47:49,046.046 INFO:tasks.util.rados:rados -p unique_pool_0 rm existed_2 2014-09-25 21:47:49,046.046 INFO:teuthology.orchestra.run.vpm057:Running: 'adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage rados -p unique_pool_0 rm existed_2' 2014-09-25 21:47:49,621.621 INFO:tasks.util.rados:rados -p unique_pool_0 put existing_3 /etc/fstab 2014-09-25 21:47:49,622.622 INFO:teuthology.orchestra.run.vpm057:Running: 'adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage rados -p unique_pool_0 put existing_3 /etc/fstab' 2014-09-25 21:47:50,429.429 INFO:tasks.util.rados:rados -p unique_pool_0 put existed_3 /etc/fstab 2014-09-25 21:47:50,429.429 INFO:teuthology.orchestra.run.vpm057:Running: 'adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage rados -p unique_pool_0 put existed_3 /etc/fstab' 2014-09-25 21:47:51,284.284 INFO:tasks.util.rados:rados -p unique_pool_0 rm existed_3 2014-09-25 21:47:51,284.284 INFO:teuthology.orchestra.run.vpm057:Running: 'adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage rados -p unique_pool_0 rm existed_3' 2014-09-25 21:47:51,789.789 INFO:tasks.util.rados:rados -p unique_pool_0 put existing_4 /etc/fstab 2014-09-25 21:47:51,789.789 INFO:teuthology.orchestra.run.vpm057:Running: 'adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage rados -p unique_pool_0 put existing_4 /etc/fstab' 2014-09-25 21:47:52,613.613 INFO:tasks.util.rados:rados -p unique_pool_0 put existed_4 /etc/fstab 2014-09-25 21:47:52,613.613 INFO:teuthology.orchestra.run.vpm057:Running: 'adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage rados -p unique_pool_0 put existed_4 /etc/fstab' 2014-09-25 21:47:53,286.286 INFO:tasks.util.rados:rados -p unique_pool_0 rm existed_4 2014-09-25 21:47:53,286.286 INFO:teuthology.orchestra.run.vpm057:Running: 'adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage rados -p unique_pool_0 rm existed_4' 2014-09-25 21:47:54,026.026 INFO:tasks.util.rados:rados -p unique_pool_0 put existing_5 /etc/fstab 2014-09-25 21:47:54,026.026 INFO:teuthology.orchestra.run.vpm057:Running: 'adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage rados -p unique_pool_0 put existing_5 /etc/fstab' 2014-09-25 21:47:54,535.535 INFO:tasks.util.rados:rados -p unique_pool_0 put existed_5 /etc/fstab 2014-09-25 21:47:54,535.535 INFO:teuthology.orchestra.run.vpm057:Running: 'adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage rados -p unique_pool_0 put existed_5 /etc/fstab' 2014-09-25 21:47:55,554.554 INFO:tasks.util.rados:rados -p unique_pool_0 rm existed_5 2014-09-25 21:47:55,554.554 INFO:teuthology.orchestra.run.vpm057:Running: 'adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage rados -p unique_pool_0 rm existed_5' 2014-09-25 21:47:56,256.256 INFO:tasks.util.rados:rados -p unique_pool_0 put existing_6 /etc/fstab 2014-09-25 21:47:56,256.256 INFO:teuthology.orchestra.run.vpm057:Running: 'adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage rados -p unique_pool_0 put existing_6 /etc/fstab' 2014-09-25 21:47:56,777.777 INFO:tasks.util.rados:rados -p unique_pool_0 put existed_6 /etc/fstab 2014-09-25 21:47:56,778.778 INFO:teuthology.orchestra.run.vpm057:Running: 'adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage rados -p unique_pool_0 put existed_6 /etc/fstab' 2014-09-25 21:47:57,518.518 INFO:tasks.util.rados:rados -p unique_pool_0 rm existed_6 2014-09-25 21:47:57,518.518 INFO:teuthology.orchestra.run.vpm057:Running: 'adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage rados -p unique_pool_0 rm existed_6' 2014-09-25 21:47:58,063.063 INFO:tasks.util.rados:rados -p unique_pool_0 put existing_7 /etc/fstab 2014-09-25 21:47:58,063.063 INFO:teuthology.orchestra.run.vpm057:Running: 'adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage rados -p unique_pool_0 put existing_7 /etc/fstab' 2014-09-25 21:47:58,916.916 INFO:tasks.util.rados:rados -p unique_pool_0 put existed_7 /etc/fstab 2014-09-25 21:47:58,916.916 INFO:teuthology.orchestra.run.vpm057:Running: 'adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage rados -p unique_pool_0 put existed_7 /etc/fstab' 2014-09-25 21:47:59,462.462 INFO:tasks.util.rados:rados -p unique_pool_0 rm existed_7 2014-09-25 21:47:59,462.462 INFO:teuthology.orchestra.run.vpm057:Running: 'adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage rados -p unique_pool_0 rm existed_7' 2014-09-25 21:47:59,984.984 INFO:tasks.util.rados:rados -p unique_pool_0 put existing_8 /etc/fstab 2014-09-25 21:47:59,985.985 INFO:teuthology.orchestra.run.vpm057:Running: 'adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage rados -p unique_pool_0 put existing_8 /etc/fstab' 2014-09-25 21:48:00,885.885 INFO:tasks.util.rados:rados -p unique_pool_0 put existed_8 /etc/fstab 2014-09-25 21:48:00,886.886 INFO:teuthology.orchestra.run.vpm057:Running: 'adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage rados -p unique_pool_0 put existed_8 /etc/fstab' 2014-09-25 21:48:01,601.601 INFO:tasks.util.rados:rados -p unique_pool_0 rm existed_8 2014-09-25 21:48:01,601.601 INFO:teuthology.orchestra.run.vpm057:Running: 'adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage rados -p unique_pool_0 rm existed_8' 2014-09-25 21:48:02,310.310 INFO:tasks.util.rados:rados -p unique_pool_0 put existing_9 /etc/fstab 2014-09-25 21:48:02,310.310 INFO:teuthology.orchestra.run.vpm057:Running: 'adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage rados -p unique_pool_0 put existing_9 /etc/fstab' 2014-09-25 21:48:03,153.153 INFO:tasks.util.rados:rados -p unique_pool_0 put existed_9 /etc/fstab 2014-09-25 21:48:03,153.153 INFO:teuthology.orchestra.run.vpm057:Running: 'adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage rados -p unique_pool_0 put existed_9 /etc/fstab' 2014-09-25 21:48:03,828.828 INFO:tasks.util.rados:rados -p unique_pool_0 rm existed_9 2014-09-25 21:48:03,828.828 INFO:teuthology.orchestra.run.vpm057:Running: 'adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage rados -p unique_pool_0 rm existed_9' 2014-09-25 21:48:04,441.441 INFO:teuthology.orchestra.run.vpm057:Running: "adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph tell osd.1 injectargs '--osd-recovery-delay-start 1000 --osd-min-pg-log-entries 100000000'" 2014-09-25 21:48:05,209.209 INFO:teuthology.orchestra.run.vpm057.stderr:osd_min_pg_log_entries = '100000000' osd_recovery_delay_start = '1000' 2014-09-25 21:48:11,221.221 INFO:tasks.ceph.osd.0:Stopped 2014-09-25 21:48:11,221.221 INFO:teuthology.orchestra.run.vpm057:Running: 'adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph osd down 0' 2014-09-25 21:48:11,885.885 INFO:teuthology.orchestra.run.vpm057.stderr:marked down osd.0. 2014-09-25 21:48:17,917.917 INFO:tasks.ceph.osd.3:Stopped 2014-09-25 21:48:17,917.917 INFO:teuthology.orchestra.run.vpm057:Running: 'adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph osd down 3' 2014-09-25 21:48:19,502.502 INFO:teuthology.orchestra.run.vpm057.stderr:marked down osd.3. 2014-09-25 21:48:19,508.508 INFO:tasks.util.rados:rados -p unique_pool_0 put new_1 /etc/fstab 2014-09-25 21:48:19,508.508 INFO:teuthology.orchestra.run.vpm057:Running: 'adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage rados -p unique_pool_0 put new_1 /etc/fstab' 2014-09-25 21:48:20,649.649 INFO:tasks.util.rados:rados -p unique_pool_0 put existed_1 /etc/fstab 2014-09-25 21:48:20,649.649 INFO:teuthology.orchestra.run.vpm057:Running: 'adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage rados -p unique_pool_0 put existed_1 /etc/fstab' 2014-09-25 21:48:21,367.367 INFO:tasks.util.rados:rados -p unique_pool_0 put existing_1 /etc/fstab 2014-09-25 21:48:21,367.367 INFO:teuthology.orchestra.run.vpm057:Running: 'adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage rados -p unique_pool_0 put existing_1 /etc/fstab' 2014-09-25 21:48:21,937.937 INFO:tasks.util.rados:rados -p unique_pool_0 put new_2 /etc/fstab 2014-09-25 21:48:21,937.937 INFO:teuthology.orchestra.run.vpm057:Running: 'adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage rados -p unique_pool_0 put new_2 /etc/fstab' 2014-09-25 21:48:22,695.695 INFO:tasks.util.rados:rados -p unique_pool_0 put existed_2 /etc/fstab 2014-09-25 21:48:22,696.696 INFO:teuthology.orchestra.run.vpm057:Running: 'adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage rados -p unique_pool_0 put existed_2 /etc/fstab' 2014-09-25 21:48:23,619.619 INFO:tasks.util.rados:rados -p unique_pool_0 put existing_2 /etc/fstab 2014-09-25 21:48:23,619.619 INFO:teuthology.orchestra.run.vpm057:Running: 'adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage rados -p unique_pool_0 put existing_2 /etc/fstab' 2014-09-25 21:48:24,336.336 INFO:tasks.util.rados:rados -p unique_pool_0 put new_3 /etc/fstab 2014-09-25 21:48:24,337.337 INFO:teuthology.orchestra.run.vpm057:Running: 'adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage rados -p unique_pool_0 put new_3 /etc/fstab' 2014-09-25 21:48:24,909.909 INFO:tasks.util.rados:rados -p unique_pool_0 put existed_3 /etc/fstab 2014-09-25 21:48:24,910.910 INFO:teuthology.orchestra.run.vpm057:Running: 'adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage rados -p unique_pool_0 put existed_3 /etc/fstab' 2014-09-25 21:48:25,769.769 INFO:tasks.util.rados:rados -p unique_pool_0 put existing_3 /etc/fstab 2014-09-25 21:48:25,769.769 INFO:teuthology.orchestra.run.vpm057:Running: 'adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage rados -p unique_pool_0 put existing_3 /etc/fstab' 2014-09-25 21:48:26,486.486 INFO:tasks.util.rados:rados -p unique_pool_0 put new_4 /etc/fstab 2014-09-25 21:48:26,486.486 INFO:teuthology.orchestra.run.vpm057:Running: 'adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage rados -p unique_pool_0 put new_4 /etc/fstab' 2014-09-25 21:48:27,103.103 INFO:tasks.util.rados:rados -p unique_pool_0 put existed_4 /etc/fstab 2014-09-25 21:48:27,103.103 INFO:teuthology.orchestra.run.vpm057:Running: 'adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage rados -p unique_pool_0 put existed_4 /etc/fstab' 2014-09-25 21:48:27,819.819 INFO:tasks.util.rados:rados -p unique_pool_0 put existing_4 /etc/fstab 2014-09-25 21:48:27,819.819 INFO:teuthology.orchestra.run.vpm057:Running: 'adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage rados -p unique_pool_0 put existing_4 /etc/fstab' 2014-09-25 21:48:28,489.489 INFO:tasks.util.rados:rados -p unique_pool_0 put new_5 /etc/fstab 2014-09-25 21:48:28,489.489 INFO:teuthology.orchestra.run.vpm057:Running: 'adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage rados -p unique_pool_0 put new_5 /etc/fstab' 2014-09-25 21:48:29,268.268 INFO:tasks.util.rados:rados -p unique_pool_0 put existed_5 /etc/fstab 2014-09-25 21:48:29,269.269 INFO:teuthology.orchestra.run.vpm057:Running: 'adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage rados -p unique_pool_0 put existed_5 /etc/fstab' 2014-09-25 21:48:30,107.107 INFO:tasks.util.rados:rados -p unique_pool_0 put existing_5 /etc/fstab 2014-09-25 21:48:30,107.107 INFO:teuthology.orchestra.run.vpm057:Running: 'adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage rados -p unique_pool_0 put existing_5 /etc/fstab' 2014-09-25 21:48:30,992.992 INFO:tasks.util.rados:rados -p unique_pool_0 put new_6 /etc/fstab 2014-09-25 21:48:30,993.993 INFO:teuthology.orchestra.run.vpm057:Running: 'adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage rados -p unique_pool_0 put new_6 /etc/fstab' 2014-09-25 21:48:31,685.685 INFO:tasks.util.rados:rados -p unique_pool_0 put existed_6 /etc/fstab 2014-09-25 21:48:31,686.686 INFO:teuthology.orchestra.run.vpm057:Running: 'adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage rados -p unique_pool_0 put existed_6 /etc/fstab' 2014-09-25 21:48:32,174.174 INFO:tasks.util.rados:rados -p unique_pool_0 put existing_6 /etc/fstab 2014-09-25 21:48:32,175.175 INFO:teuthology.orchestra.run.vpm057:Running: 'adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage rados -p unique_pool_0 put existing_6 /etc/fstab' 2014-09-25 21:48:33,242.242 INFO:tasks.util.rados:rados -p unique_pool_0 put new_7 /etc/fstab 2014-09-25 21:48:33,242.242 INFO:teuthology.orchestra.run.vpm057:Running: 'adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage rados -p unique_pool_0 put new_7 /etc/fstab' 2014-09-25 21:48:33,963.963 INFO:tasks.util.rados:rados -p unique_pool_0 put existed_7 /etc/fstab 2014-09-25 21:48:33,964.964 INFO:teuthology.orchestra.run.vpm057:Running: 'adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage rados -p unique_pool_0 put existed_7 /etc/fstab' 2014-09-25 21:48:34,574.574 INFO:tasks.util.rados:rados -p unique_pool_0 put existing_7 /etc/fstab 2014-09-25 21:48:34,574.574 INFO:teuthology.orchestra.run.vpm057:Running: 'adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage rados -p unique_pool_0 put existing_7 /etc/fstab' 2014-09-25 21:48:35,395.395 INFO:tasks.util.rados:rados -p unique_pool_0 put new_8 /etc/fstab 2014-09-25 21:48:35,396.396 INFO:teuthology.orchestra.run.vpm057:Running: 'adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage rados -p unique_pool_0 put new_8 /etc/fstab' 2014-09-25 21:48:36,127.127 INFO:tasks.util.rados:rados -p unique_pool_0 put existed_8 /etc/fstab 2014-09-25 21:48:36,127.127 INFO:teuthology.orchestra.run.vpm057:Running: 'adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage rados -p unique_pool_0 put existed_8 /etc/fstab' 2014-09-25 21:48:36,659.659 INFO:tasks.util.rados:rados -p unique_pool_0 put existing_8 /etc/fstab 2014-09-25 21:48:36,659.659 INFO:teuthology.orchestra.run.vpm057:Running: 'adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage rados -p unique_pool_0 put existing_8 /etc/fstab' 2014-09-25 21:48:37,569.569 INFO:tasks.util.rados:rados -p unique_pool_0 put new_9 /etc/fstab 2014-09-25 21:48:37,569.569 INFO:teuthology.orchestra.run.vpm057:Running: 'adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage rados -p unique_pool_0 put new_9 /etc/fstab' 2014-09-25 21:48:38,180.180 INFO:tasks.util.rados:rados -p unique_pool_0 put existed_9 /etc/fstab 2014-09-25 21:48:38,180.180 INFO:teuthology.orchestra.run.vpm057:Running: 'adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage rados -p unique_pool_0 put existed_9 /etc/fstab' 2014-09-25 21:48:38,877.877 INFO:tasks.util.rados:rados -p unique_pool_0 put existing_9 /etc/fstab 2014-09-25 21:48:38,878.878 INFO:teuthology.orchestra.run.vpm057:Running: 'adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage rados -p unique_pool_0 put existing_9 /etc/fstab' 2014-09-25 21:48:45,484.484 INFO:tasks.ceph.osd.1:Stopped 2014-09-25 21:48:45,484.484 INFO:teuthology.orchestra.run.vpm057:Running: 'adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph osd down 1' 2014-09-25 21:48:46,114.114 INFO:teuthology.orchestra.run.vpm057.stderr:marked down osd.1. 2014-09-25 21:48:46,114.114 INFO:teuthology.orchestra.run.vpm057:Running: 'adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph osd lost 1 --yes-i-really-mean-it' 2014-09-25 21:48:47,120.120 INFO:teuthology.orchestra.run.vpm057.stderr:marked osd lost in epoch 15 2014-09-25 21:48:47,128.128 INFO:tasks.ceph.osd.0:Restarting daemon 2014-09-25 21:48:47,128.128 INFO:teuthology.orchestra.run.vpm057:Running: 'sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage daemon-helper kill ceph-osd -f -i 0' 2014-09-25 21:48:47,608.608 INFO:tasks.ceph.osd.0:Started 2014-09-25 21:48:47,608.608 INFO:teuthology.orchestra.run.vpm057:Running: 'sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph --admin-daemon /var/run/ceph/ceph-osd.0.asok dump_ops_in_flight' 2014-09-25 21:48:47,656.656 INFO:tasks.ceph.osd.0.vpm057.stdout:starting osd.0 at :/0 osd_data /var/lib/ceph/osd/ceph-0 /var/lib/ceph/osd/ceph-0/journal 2014-09-25 21:48:47,989.989 INFO:tasks.ceph.osd.0.vpm057.stderr:2014-09-25 19:48:47.812365 7fe5c9942900 -1 journal FileJournal::_open: disabling aio for non-block journal. Use journal_force_aio to force use of aio anyway 2014-09-25 21:48:48,090.090 INFO:tasks.ec_lost_unfound.ceph_manager:waiting for osd.0 to be up 2014-09-25 21:48:48,091.091 INFO:teuthology.orchestra.run.vpm057:Running: 'adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph osd dump --format=json' 2014-09-25 21:48:51,781.781 INFO:tasks.ec_lost_unfound.ceph_manager:osd.0 is up 2014-09-25 21:48:51,782.782 INFO:tasks.ceph.osd.3:Restarting daemon 2014-09-25 21:48:51,782.782 INFO:teuthology.orchestra.run.vpm057:Running: 'sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage daemon-helper kill ceph-osd -f -i 3' 2014-09-25 21:48:52,495.495 INFO:tasks.ceph.osd.3:Started 2014-09-25 21:48:52,496.496 INFO:teuthology.orchestra.run.vpm057:Running: 'sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph --admin-daemon /var/run/ceph/ceph-osd.3.asok dump_ops_in_flight' 2014-09-25 21:48:52,497.497 INFO:tasks.ceph.osd.3.vpm057.stdout:starting osd.3 at :/0 osd_data /var/lib/ceph/osd/ceph-3 /var/lib/ceph/osd/ceph-3/journal 2014-09-25 21:48:52,801.801 INFO:tasks.ceph.osd.3.vpm057.stderr:2014-09-25 19:48:52.558900 7f5f6ca77900 -1 journal FileJournal::_open: disabling aio for non-block journal. Use journal_force_aio to force use of aio anyway 2014-09-25 21:48:53,109.109 INFO:tasks.ec_lost_unfound.ceph_manager:waiting for osd.3 to be up 2014-09-25 21:48:53,109.109 INFO:teuthology.orchestra.run.vpm057:Running: 'adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph osd dump --format=json' 2014-09-25 21:48:54,122.122 INFO:tasks.ec_lost_unfound.ceph_manager:osd.3 is up 2014-09-25 21:48:54,122.122 INFO:teuthology.orchestra.run.vpm057:Running: 'adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph tell osd.0 flush_pg_stats' 2014-09-25 21:48:54,934.934 INFO:teuthology.orchestra.run.vpm057:Running: 'adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph tell osd.2 flush_pg_stats' 2014-09-25 21:48:55,990.990 INFO:teuthology.orchestra.run.vpm057:Running: 'adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph tell osd.3 flush_pg_stats' 2014-09-25 21:48:57,829.829 INFO:tasks.ec_lost_unfound.ceph_manager:waiting till active 2014-09-25 21:48:57,830.830 INFO:teuthology.orchestra.run.vpm057:Running: 'adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph pg dump --format=json' 2014-09-25 21:48:58,638.638 INFO:teuthology.orchestra.run.vpm057.stderr:dumped all in format json 2014-09-25 21:48:59,456.456 INFO:teuthology.orchestra.run.vpm057:Running: 'adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph status --format=json-pretty' 2014-09-25 21:49:00,005.005 INFO:tasks.ec_lost_unfound.ceph_manager:{u'election_epoch': 4, u'quorum': [0, 1, 2], u'mdsmap': {u'max': 0, u'epoch': 1, u'by_rank': [], u'up': 0, u'in': 0}, u'monmap': {u'epoch': 1, u'mons': [{u'name': u'a', u'rank': 0, u'addr': u'10.214.138.97:6789/0'}, {u'name': u'b', u'rank': 1, u'addr': u'10.214.138.97:6790/0'}, {u'name': u'c', u'rank': 2, u'addr': u'10.214.138.97:6791/0'}], u'modified': u'2014-09-25 19:46:21.446598', u'fsid': u'a0af5d53-05bd-40c8-9d30-7782574e4764', u'created': u'2014-09-25 19:46:21.446598'}, u'health': {u'detail': [], u'timechecks': {u'round_status': u'finished', u'epoch': 4, u'round': 2, u'mons': [{u'latency': u'0.000000', u'skew': u'0.000000', u'health': u'HEALTH_OK', u'name': u'a'}, {u'latency': u'0.003394', u'skew': u'0.000000', u'health': u'HEALTH_OK', u'name': u'b'}, {u'latency': u'0.018347', u'skew': u'0.000000', u'health': u'HEALTH_OK', u'name': u'c'}]}, u'health': {u'health_services': [{u'mons': [{u'last_updated': u'2014-09-25 19:48:02.482279', u'name': u'a', u'avail_percent': 91, u'kb_total': 103178528, u'kb_avail': 93900220, u'health': u'HEALTH_OK', u'kb_used': 5053348, u'store_stats': {u'bytes_total': 919659, u'bytes_log': 914056, u'last_updated': u'0.000000', u'bytes_misc': 5603, u'bytes_sst': 0}}, {u'last_updated': u'2014-09-25 19:48:02.838418', u'name': u'b', u'avail_percent': 91, u'kb_total': 103178528, u'kb_avail': 93900244, u'health': u'HEALTH_OK', u'kb_used': 5053324, u'store_stats': {u'bytes_total': 1353185, u'bytes_log': 1347582, u'last_updated': u'0.000000', u'bytes_misc': 5603, u'bytes_sst': 0}}, {u'last_updated': u'2014-09-25 19:48:03.251625', u'name': u'c', u'avail_percent': 91, u'kb_total': 103178528, u'kb_avail': 93900168, u'health': u'HEALTH_OK', u'kb_used': 5053400, u'store_stats': {u'bytes_total': 1352705, u'bytes_log': 1347102, u'last_updated': u'0.000000', u'bytes_misc': 5603, u'bytes_sst': 0}}]}]}, u'overall_status': u'HEALTH_WARN', u'summary': [{u'severity': u'HEALTH_WARN', u'summary': u'21 pgs degraded'}, {u'severity': u'HEALTH_WARN', u'summary': u'3 pgs stuck unclean'}, {u'severity': u'HEALTH_WARN', u'summary': u'7 pgs undersized'}, {u'severity': u'HEALTH_WARN', u'summary': u'recovery 65/112 objects degraded (58.036%); 28/112 objects misplaced (25.000%); 27/28 unfound (96.429%)'}, {u'severity': u'HEALTH_WARN', u'summary': u'1/4 in osds are down'}]}, u'pgmap': {u'bytes_total': 422619250688, u'degraded_objects': 65, u'num_pgs': 32, u'misplaced_objects': 28, u'data_bytes': 1204, u'degraded_total': 112, u'bytes_used': 20721364992, u'unfound_ratio': u'96.429', u'op_per_sec': 1, u'write_bytes_sec': 2003, u'unfound_total': 28, u'unfound_objects': 27, u'version': 49, u'pgs_by_state': [{u'count': 7, u'state_name': u'active+undersized+degraded'}, {u'count': 14, u'state_name': u'active+degraded'}, {u'count': 11, u'state_name': u'active+clean'}], u'misplaced_ratio': u'25.000', u'degraded_ratio': u'58.036', u'bytes_avail': 384592449536, u'misplaced_total': 112}, u'quorum_names': [u'a', u'b', u'c'], u'osdmap': {u'osdmap': {u'full': False, u'nearfull': False, u'num_osds': 4, u'num_up_osds': 3, u'epoch': 20, u'num_in_osds': 4}}, u'fsid': u'a0af5d53-05bd-40c8-9d30-7782574e4764'} 2014-09-25 21:49:00,006.006 INFO:tasks.ec_lost_unfound.ceph_manager:active! 2014-09-25 21:49:00,006.006 INFO:teuthology.orchestra.run.vpm057:Running: 'adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph tell osd.0 flush_pg_stats' 2014-09-25 21:49:00,688.688 INFO:teuthology.orchestra.run.vpm057:Running: 'adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph tell osd.2 flush_pg_stats' 2014-09-25 21:49:01,171.171 INFO:teuthology.orchestra.run.vpm057:Running: 'adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph tell osd.3 flush_pg_stats' 2014-09-25 21:49:01,963.963 INFO:teuthology.orchestra.run.vpm057:Running: 'adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph status --format=json-pretty' 2014-09-25 21:49:02,626.626 INFO:tasks.ec_lost_unfound.ceph_manager:{u'election_epoch': 4, u'quorum': [0, 1, 2], u'mdsmap': {u'max': 0, u'epoch': 1, u'by_rank': [], u'up': 0, u'in': 0}, u'monmap': {u'epoch': 1, u'mons': [{u'name': u'a', u'rank': 0, u'addr': u'10.214.138.97:6789/0'}, {u'name': u'b', u'rank': 1, u'addr': u'10.214.138.97:6790/0'}, {u'name': u'c', u'rank': 2, u'addr': u'10.214.138.97:6791/0'}], u'modified': u'2014-09-25 19:46:21.446598', u'fsid': u'a0af5d53-05bd-40c8-9d30-7782574e4764', u'created': u'2014-09-25 19:46:21.446598'}, u'health': {u'detail': [], u'timechecks': {u'round_status': u'finished', u'epoch': 4, u'round': 2, u'mons': [{u'latency': u'0.000000', u'skew': u'0.000000', u'health': u'HEALTH_OK', u'name': u'a'}, {u'latency': u'0.003394', u'skew': u'0.000000', u'health': u'HEALTH_OK', u'name': u'b'}, {u'latency': u'0.018347', u'skew': u'0.000000', u'health': u'HEALTH_OK', u'name': u'c'}]}, u'health': {u'health_services': [{u'mons': [{u'last_updated': u'2014-09-25 19:48:02.482279', u'name': u'a', u'avail_percent': 91, u'kb_total': 103178528, u'kb_avail': 93900220, u'health': u'HEALTH_OK', u'kb_used': 5053348, u'store_stats': {u'bytes_total': 919659, u'bytes_log': 914056, u'last_updated': u'0.000000', u'bytes_misc': 5603, u'bytes_sst': 0}}, {u'last_updated': u'2014-09-25 19:48:02.838418', u'name': u'b', u'avail_percent': 91, u'kb_total': 103178528, u'kb_avail': 93900244, u'health': u'HEALTH_OK', u'kb_used': 5053324, u'store_stats': {u'bytes_total': 1353185, u'bytes_log': 1347582, u'last_updated': u'0.000000', u'bytes_misc': 5603, u'bytes_sst': 0}}, {u'last_updated': u'2014-09-25 19:48:03.251625', u'name': u'c', u'avail_percent': 91, u'kb_total': 103178528, u'kb_avail': 93900168, u'health': u'HEALTH_OK', u'kb_used': 5053400, u'store_stats': {u'bytes_total': 1352705, u'bytes_log': 1347102, u'last_updated': u'0.000000', u'bytes_misc': 5603, u'bytes_sst': 0}}]}]}, u'overall_status': u'HEALTH_WARN', u'summary': [{u'severity': u'HEALTH_WARN', u'summary': u'21 pgs degraded'}, {u'severity': u'HEALTH_WARN', u'summary': u'3 pgs stuck unclean'}, {u'severity': u'HEALTH_WARN', u'summary': u'7 pgs undersized'}, {u'severity': u'HEALTH_WARN', u'summary': u'recovery 65/112 objects degraded (58.036%); 28/112 objects misplaced (25.000%); 27/28 unfound (96.429%)'}, {u'severity': u'HEALTH_WARN', u'summary': u'1/4 in osds are down'}]}, u'pgmap': {u'bytes_total': 422619250688, u'degraded_objects': 65, u'num_pgs': 32, u'misplaced_objects': 28, u'data_bytes': 1204, u'degraded_total': 112, u'bytes_used': 20721364992, u'unfound_ratio': u'96.429', u'op_per_sec': 1, u'write_bytes_sec': 2003, u'unfound_total': 28, u'unfound_objects': 27, u'version': 49, u'pgs_by_state': [{u'count': 7, u'state_name': u'active+undersized+degraded'}, {u'count': 14, u'state_name': u'active+degraded'}, {u'count': 11, u'state_name': u'active+clean'}], u'misplaced_ratio': u'25.000', u'degraded_ratio': u'58.036', u'bytes_avail': 384592449536, u'misplaced_total': 112}, u'quorum_names': [u'a', u'b', u'c'], u'osdmap': {u'osdmap': {u'full': False, u'nearfull': False, u'num_osds': 4, u'num_up_osds': 3, u'epoch': 20, u'num_in_osds': 4}}, u'fsid': u'a0af5d53-05bd-40c8-9d30-7782574e4764'} 2014-09-25 21:49:02,627.627 INFO:tasks.ec_lost_unfound:there are 27 unfound objects 2014-09-25 21:49:02,628.628 INFO:teuthology.orchestra.run.vpm057:Running: 'adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph pg dump --format=json' 2014-09-25 21:49:03,470.470 INFO:teuthology.orchestra.run.vpm057.stderr:dumped all in format json 2014-09-25 21:49:04,235.235 INFO:tasks.ec_lost_unfound:listing missing/lost in 1.e state active+degraded 2014-09-25 21:49:04,236.236 INFO:teuthology.orchestra.run.vpm057:Running: u"adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph -- pg 1.e list_missing '{}'" 2014-09-25 21:49:08,980.980 INFO:tasks.ec_lost_unfound:{u'num_unfound': 2, u'objects': [{u'need': u"14'3", u'oid': {u'hash': 1224592654, u'max': 0, u'oid': u'existing_4', u'namespace': u'', u'snapid': -2, u'key': u'', u'pool': 1}, u'locations': [u'2(3)'], u'have': u"10'1"}, {u'need': u"14'4", u'oid': {u'hash': 3101443614, u'max': 0, u'oid': u'existing_6', u'namespace': u'', u'snapid': -2, u'key': u'', u'pool': 1}, u'locations': [u'2(3)'], u'have': u"10'2"}], u'num_missing': 2, u'offset': {u'hash': 0, u'max': 0, u'oid': u'', u'namespace': u'', u'snapid': 0, u'key': u'', u'pool': -1}} 2014-09-25 21:49:08,980.980 INFO:tasks.ec_lost_unfound:reverting unfound in 1.e 2014-09-25 21:49:08,981.981 INFO:teuthology.orchestra.run.vpm057:Running: u'adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph pg 1.e mark_unfound_lost delete' 2014-09-25 21:49:09,571.571 INFO:teuthology.orchestra.run.vpm057.stderr:pg has 2 objects unfound and apparently lost, marking 2014-09-25 21:49:09,585.585 INFO:tasks.ec_lost_unfound:no unfound in 0.f 2014-09-25 21:49:09,585.585 INFO:tasks.ec_lost_unfound:no unfound in 1.f 2014-09-25 21:49:09,585.585 INFO:tasks.ec_lost_unfound:no unfound in 0.e 2014-09-25 21:49:09,585.585 INFO:tasks.ec_lost_unfound:no unfound in 1.c 2014-09-25 21:49:09,585.585 INFO:tasks.ec_lost_unfound:no unfound in 0.d 2014-09-25 21:49:09,585.585 INFO:tasks.ec_lost_unfound:listing missing/lost in 1.d state active+degraded 2014-09-25 21:49:09,585.585 INFO:teuthology.orchestra.run.vpm057:Running: u"adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph -- pg 1.d list_missing '{}'" 2014-09-25 21:49:10,219.219 INFO:tasks.ec_lost_unfound:{u'num_unfound': 1, u'objects': [], u'num_missing': 0, u'offset': {u'hash': 0, u'max': 0, u'oid': u'', u'namespace': u'', u'snapid': 0, u'key': u'', u'pool': -1}} 2014-09-25 21:49:10,220.220 INFO:tasks.ec_lost_unfound:reverting unfound in 1.d 2014-09-25 21:49:10,220.220 INFO:teuthology.orchestra.run.vpm057:Running: u'adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph pg 1.d mark_unfound_lost delete' 2014-09-25 21:49:10,963.963 INFO:teuthology.orchestra.run.vpm057.stderr:pg has 1 objects unfound and apparently lost, marking 2014-09-25 21:49:10,991.991 INFO:tasks.ec_lost_unfound:no unfound in 0.c 2014-09-25 21:49:10,992.992 INFO:tasks.ec_lost_unfound:listing missing/lost in 1.a state active+degraded 2014-09-25 21:49:10,992.992 INFO:teuthology.orchestra.run.vpm057:Running: u"adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph -- pg 1.a list_missing '{}'" 2014-09-25 21:49:11,842.842 INFO:tasks.ec_lost_unfound:{u'num_unfound': 2, u'objects': [], u'num_missing': 0, u'offset': {u'hash': 0, u'max': 0, u'oid': u'', u'namespace': u'', u'snapid': 0, u'key': u'', u'pool': -1}} 2014-09-25 21:49:11,842.842 INFO:tasks.ec_lost_unfound:reverting unfound in 1.a 2014-09-25 21:49:11,843.843 INFO:teuthology.orchestra.run.vpm057:Running: u'adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph pg 1.a mark_unfound_lost delete' 2014-09-25 21:49:12,462.462 INFO:teuthology.orchestra.run.vpm057.stderr:pg has 2 objects unfound and apparently lost, marking 2014-09-25 21:49:12,463.463 INFO:tasks.ec_lost_unfound:no unfound in 0.b 2014-09-25 21:49:12,463.463 INFO:tasks.ec_lost_unfound:listing missing/lost in 1.b state active+degraded 2014-09-25 21:49:12,463.463 INFO:teuthology.orchestra.run.vpm057:Running: u"adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph -- pg 1.b list_missing '{}'" 2014-09-25 21:49:13,354.354 INFO:tasks.ec_lost_unfound:{u'num_unfound': 2, u'objects': [{u'need': u"14'4", u'oid': {u'hash': 2885884443, u'max': 0, u'oid': u'existing_1', u'namespace': u'', u'snapid': -2, u'key': u'', u'pool': 1}, u'locations': [u'2(2)'], u'have': u"10'1"}, {u'need': u"14'5", u'oid': {u'hash': 943960171, u'max': 0, u'oid': u'existed_3', u'namespace': u'', u'snapid': -2, u'key': u'', u'pool': 1}, u'locations': [u'2(2)'], u'have': u"0'0"}], u'num_missing': 2, u'offset': {u'hash': 0, u'max': 0, u'oid': u'', u'namespace': u'', u'snapid': 0, u'key': u'', u'pool': -1}} 2014-09-25 21:49:13,354.354 INFO:tasks.ec_lost_unfound:reverting unfound in 1.b 2014-09-25 21:49:13,354.354 INFO:teuthology.orchestra.run.vpm057:Running: u'adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph pg 1.b mark_unfound_lost delete' 2014-09-25 21:49:13,998.998 INFO:teuthology.orchestra.run.vpm057.stderr:pg has 2 objects unfound and apparently lost, marking 2014-09-25 21:49:14,006.006 INFO:tasks.ec_lost_unfound:no unfound in 0.a 2014-09-25 21:49:14,006.006 INFO:tasks.ec_lost_unfound:listing missing/lost in 1.8 state active+degraded 2014-09-25 21:49:14,006.006 INFO:teuthology.orchestra.run.vpm057:Running: u"adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph -- pg 1.8 list_missing '{}'" 2014-09-25 21:49:14,820.820 INFO:tasks.ec_lost_unfound:{u'num_unfound': 1, u'objects': [], u'num_missing': 0, u'offset': {u'hash': 0, u'max': 0, u'oid': u'', u'namespace': u'', u'snapid': 0, u'key': u'', u'pool': -1}} 2014-09-25 21:49:14,820.820 INFO:tasks.ec_lost_unfound:reverting unfound in 1.8 2014-09-25 21:49:14,821.821 INFO:teuthology.orchestra.run.vpm057:Running: u'adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph pg 1.8 mark_unfound_lost delete' 2014-09-25 21:49:15,467.467 INFO:teuthology.orchestra.run.vpm057.stderr:pg has 1 objects unfound and apparently lost, marking 2014-09-25 21:49:15,484.484 INFO:tasks.ec_lost_unfound:no unfound in 0.9 2014-09-25 21:49:15,484.484 INFO:tasks.ec_lost_unfound:listing missing/lost in 1.9 state active+degraded 2014-09-25 21:49:15,484.484 INFO:teuthology.orchestra.run.vpm057:Running: u"adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph -- pg 1.9 list_missing '{}'" 2014-09-25 21:49:16,128.128 INFO:tasks.ec_lost_unfound:{u'num_unfound': 2, u'objects': [{u'need': u"14'4", u'oid': {u'hash': 2998549577, u'max': 0, u'oid': u'existing_5', u'namespace': u'', u'snapid': -2, u'key': u'', u'pool': 1}, u'locations': [u'2(2)'], u'have': u"10'2"}, {u'need': u"14'3", u'oid': {u'hash': 4032093161, u'max': 0, u'oid': u'existing_2', u'namespace': u'', u'snapid': -2, u'key': u'', u'pool': 1}, u'locations': [u'2(2)'], u'have': u"10'1"}], u'num_missing': 2, u'offset': {u'hash': 0, u'max': 0, u'oid': u'', u'namespace': u'', u'snapid': 0, u'key': u'', u'pool': -1}} 2014-09-25 21:49:16,129.129 INFO:tasks.ec_lost_unfound:reverting unfound in 1.9 2014-09-25 21:49:16,129.129 INFO:teuthology.orchestra.run.vpm057:Running: u'adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph pg 1.9 mark_unfound_lost delete' 2014-09-25 21:49:16,746.746 INFO:teuthology.orchestra.run.vpm057.stderr:pg has 2 objects unfound and apparently lost, marking 2014-09-25 21:49:16,757.757 INFO:tasks.ec_lost_unfound:no unfound in 0.8 2014-09-25 21:49:16,757.757 INFO:tasks.ec_lost_unfound:no unfound in 0.7 2014-09-25 21:49:16,757.757 INFO:tasks.ec_lost_unfound:listing missing/lost in 1.6 state active+degraded 2014-09-25 21:49:16,758.758 INFO:teuthology.orchestra.run.vpm057:Running: u"adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph -- pg 1.6 list_missing '{}'" 2014-09-25 21:49:17,339.339 INFO:tasks.ec_lost_unfound:{u'num_unfound': 3, u'objects': [{u'need': u"14'5", u'oid': {u'hash': 1861424966, u'max': 0, u'oid': u'existed_8', u'namespace': u'', u'snapid': -2, u'key': u'', u'pool': 1}, u'locations': [u'2(3)'], u'have': u"0'0"}, {u'need': u"14'4", u'oid': {u'hash': 3910131350, u'max': 0, u'oid': u'new_6', u'namespace': u'', u'snapid': -2, u'key': u'', u'pool': 1}, u'locations': [u'2(3)'], u'have': u"0'0"}, {u'need': u"14'3", u'oid': {u'hash': 2787839990, u'max': 0, u'oid': u'new_4', u'namespace': u'', u'snapid': -2, u'key': u'', u'pool': 1}, u'locations': [u'2(3)'], u'have': u"0'0"}], u'num_missing': 3, u'offset': {u'hash': 0, u'max': 0, u'oid': u'', u'namespace': u'', u'snapid': 0, u'key': u'', u'pool': -1}} 2014-09-25 21:49:17,339.339 INFO:tasks.ec_lost_unfound:reverting unfound in 1.6 2014-09-25 21:49:17,340.340 INFO:teuthology.orchestra.run.vpm057:Running: u'adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph pg 1.6 mark_unfound_lost delete' 2014-09-25 21:49:17,993.993 INFO:teuthology.orchestra.run.vpm057.stderr:pg has 3 objects unfound and apparently lost, marking 2014-09-25 21:49:17,997.997 INFO:tasks.ec_lost_unfound:listing missing/lost in 1.7 state active+degraded 2014-09-25 21:49:17,998.998 INFO:teuthology.orchestra.run.vpm057:Running: u"adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph -- pg 1.7 list_missing '{}'" 2014-09-25 21:49:18,639.639 INFO:tasks.ec_lost_unfound:{u'num_unfound': 1, u'objects': [{u'need': u"14'1", u'oid': {u'hash': 3761017527, u'max': 0, u'oid': u'new_7', u'namespace': u'', u'snapid': -2, u'key': u'', u'pool': 1}, u'locations': [u'2(2)'], u'have': u"0'0"}], u'num_missing': 1, u'offset': {u'hash': 0, u'max': 0, u'oid': u'', u'namespace': u'', u'snapid': 0, u'key': u'', u'pool': -1}} 2014-09-25 21:49:18,639.639 INFO:tasks.ec_lost_unfound:reverting unfound in 1.7 2014-09-25 21:49:18,639.639 INFO:teuthology.orchestra.run.vpm057:Running: u'adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph pg 1.7 mark_unfound_lost delete' 2014-09-25 21:49:19,426.426 INFO:teuthology.orchestra.run.vpm057.stderr:pg has 1 objects unfound and apparently lost, marking 2014-09-25 21:49:19,442.442 INFO:tasks.ec_lost_unfound:no unfound in 0.6 2014-09-25 21:49:19,442.442 INFO:tasks.ec_lost_unfound:no unfound in 0.5 2014-09-25 21:49:19,442.442 INFO:tasks.ec_lost_unfound:listing missing/lost in 1.4 state active+degraded 2014-09-25 21:49:19,442.442 INFO:teuthology.orchestra.run.vpm057:Running: u"adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph -- pg 1.4 list_missing '{}'" 2014-09-25 21:49:20,037.037 INFO:tasks.ec_lost_unfound:{u'num_unfound': 3, u'objects': [{u'need': u"14'7", u'oid': {u'hash': 1767667508, u'max': 0, u'oid': u'existed_2', u'namespace': u'', u'snapid': -2, u'key': u'', u'pool': 1}, u'locations': [u'2(2)'], u'have': u"0'0"}, {u'need': u"14'9", u'oid': {u'hash': 3996155460, u'max': 0, u'oid': u'existing_7', u'namespace': u'', u'snapid': -2, u'key': u'', u'pool': 1}, u'locations': [u'2(2)'], u'have': u"10'6"}, {u'need': u"14'8", u'oid': {u'hash': 873891716, u'max': 0, u'oid': u'existed_4', u'namespace': u'', u'snapid': -2, u'key': u'', u'pool': 1}, u'locations': [u'2(2)'], u'have': u"0'0"}], u'num_missing': 3, u'offset': {u'hash': 0, u'max': 0, u'oid': u'', u'namespace': u'', u'snapid': 0, u'key': u'', u'pool': -1}} 2014-09-25 21:49:20,038.038 INFO:tasks.ec_lost_unfound:reverting unfound in 1.4 2014-09-25 21:49:20,038.038 INFO:teuthology.orchestra.run.vpm057:Running: u'adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph pg 1.4 mark_unfound_lost delete' 2014-09-25 21:49:20,965.965 INFO:teuthology.orchestra.run.vpm057.stderr:pg has 3 objects unfound and apparently lost, marking 2014-09-25 21:49:20,966.966 INFO:tasks.ec_lost_unfound:no unfound in 0.4 2014-09-25 21:49:20,966.966 INFO:tasks.ec_lost_unfound:listing missing/lost in 1.5 state active+degraded 2014-09-25 21:49:20,967.967 INFO:teuthology.orchestra.run.vpm057:Running: u"adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph -- pg 1.5 list_missing '{}'" 2014-09-25 21:49:21,747.747 INFO:tasks.ec_lost_unfound:{u'num_unfound': 2, u'objects': [], u'num_missing': 0, u'offset': {u'hash': 0, u'max': 0, u'oid': u'', u'namespace': u'', u'snapid': 0, u'key': u'', u'pool': -1}} 2014-09-25 21:49:21,748.748 INFO:tasks.ec_lost_unfound:reverting unfound in 1.5 2014-09-25 21:49:21,748.748 INFO:teuthology.orchestra.run.vpm057:Running: u'adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph pg 1.5 mark_unfound_lost delete' 2014-09-25 21:49:22,324.324 INFO:teuthology.orchestra.run.vpm057.stderr:pg has 2 objects unfound and apparently lost, marking 2014-09-25 21:49:22,339.339 INFO:tasks.ec_lost_unfound:no unfound in 0.3 2014-09-25 21:49:22,339.339 INFO:tasks.ec_lost_unfound:listing missing/lost in 1.2 state active+degraded 2014-09-25 21:49:22,339.339 INFO:teuthology.orchestra.run.vpm057:Running: u"adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph -- pg 1.2 list_missing '{}'" 2014-09-25 21:49:23,114.114 INFO:tasks.ec_lost_unfound:{u'num_unfound': 2, u'objects': [{u'need': u"14'6", u'oid': {u'hash': 831402034, u'max': 0, u'oid': u'existed_9', u'namespace': u'', u'snapid': -2, u'key': u'', u'pool': 1}, u'locations': [u'2(3)'], u'have': u"0'0"}, {u'need': u"14'5", u'oid': {u'hash': 1708673474, u'max': 0, u'oid': u'existed_6', u'namespace': u'', u'snapid': -2, u'key': u'', u'pool': 1}, u'locations': [u'2(3)'], u'have': u"0'0"}], u'num_missing': 2, u'offset': {u'hash': 0, u'max': 0, u'oid': u'', u'namespace': u'', u'snapid': 0, u'key': u'', u'pool': -1}} 2014-09-25 21:49:23,114.114 INFO:tasks.ec_lost_unfound:reverting unfound in 1.2 2014-09-25 21:49:23,115.115 INFO:teuthology.orchestra.run.vpm057:Running: u'adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph pg 1.2 mark_unfound_lost delete' 2014-09-25 21:49:23,717.717 INFO:teuthology.orchestra.run.vpm057.stderr:pg has 2 objects unfound and apparently lost, marking 2014-09-25 21:49:23,728.728 INFO:tasks.ec_lost_unfound:no unfound in 0.2 2014-09-25 21:49:23,728.728 INFO:tasks.ec_lost_unfound:listing missing/lost in 1.3 state active+degraded 2014-09-25 21:49:23,729.729 INFO:teuthology.orchestra.run.vpm057:Running: u"adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph -- pg 1.3 list_missing '{}'" 2014-09-25 21:49:24,319.319 INFO:tasks.ec_lost_unfound:{u'num_unfound': 2, u'objects': [{u'need': u"14'4", u'oid': {u'hash': 3122379363, u'max': 0, u'oid': u'existed_1', u'namespace': u'', u'snapid': -2, u'key': u'', u'pool': 1}, u'locations': [u'2(2)'], u'have': u"0'0"}, {u'need': u"14'5", u'oid': {u'hash': 1901787523, u'max': 0, u'oid': u'existing_8', u'namespace': u'', u'snapid': -2, u'key': u'', u'pool': 1}, u'locations': [u'2(2)'], u'have': u"10'3"}], u'num_missing': 2, u'offset': {u'hash': 0, u'max': 0, u'oid': u'', u'namespace': u'', u'snapid': 0, u'key': u'', u'pool': -1}} 2014-09-25 21:49:24,319.319 INFO:tasks.ec_lost_unfound:reverting unfound in 1.3 2014-09-25 21:49:24,320.320 INFO:teuthology.orchestra.run.vpm057:Running: u'adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph pg 1.3 mark_unfound_lost delete' 2014-09-25 21:49:24,859.859 INFO:teuthology.orchestra.run.vpm057.stderr:pg has 2 objects unfound and apparently lost, marking 2014-09-25 21:49:24,871.871 INFO:tasks.ec_lost_unfound:no unfound in 0.1 2014-09-25 21:49:24,872.872 INFO:tasks.ec_lost_unfound:listing missing/lost in 1.0 state active+degraded 2014-09-25 21:49:24,872.872 INFO:teuthology.orchestra.run.vpm057:Running: u"adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph -- pg 1.0 list_missing '{}'" 2014-09-25 21:49:25,388.388 INFO:tasks.ec_lost_unfound:{u'num_unfound': 2, u'objects': [{u'need': u"14'3", u'oid': {u'hash': 4248945696, u'max': 0, u'oid': u'new_5', u'namespace': u'', u'snapid': -2, u'key': u'', u'pool': 1}, u'locations': [u'2(2)'], u'have': u"0'0"}, {u'need': u"14'4", u'oid': {u'hash': 4212789968, u'max': 0, u'oid': u'existed_5', u'namespace': u'', u'snapid': -2, u'key': u'', u'pool': 1}, u'locations': [u'2(2)'], u'have': u"0'0"}], u'num_missing': 2, u'offset': {u'hash': 0, u'max': 0, u'oid': u'', u'namespace': u'', u'snapid': 0, u'key': u'', u'pool': -1}} 2014-09-25 21:49:25,389.389 INFO:tasks.ec_lost_unfound:reverting unfound in 1.0 2014-09-25 21:49:25,389.389 INFO:teuthology.orchestra.run.vpm057:Running: u'adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph pg 1.0 mark_unfound_lost delete' 2014-09-25 21:49:26,237.237 INFO:teuthology.orchestra.run.vpm057.stderr:pg has 2 objects unfound and apparently lost, marking 2014-09-25 21:49:26,249.249 INFO:tasks.ec_lost_unfound:no unfound in 0.0 2014-09-25 21:49:26,249.249 INFO:tasks.ec_lost_unfound:listing missing/lost in 1.1 state active+degraded 2014-09-25 21:49:26,250.250 INFO:teuthology.orchestra.run.vpm057:Running: u"adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph -- pg 1.1 list_missing '{}'" 2014-09-25 21:49:26,990.990 INFO:tasks.ec_lost_unfound:{u'num_unfound': 2, u'objects': [{u'need': u"14'3", u'oid': {u'hash': 459768081, u'max': 0, u'oid': u'existing_3', u'namespace': u'', u'snapid': -2, u'key': u'', u'pool': 1}, u'locations': [u'2(3)'], u'have': u"10'1"}, {u'need': u"14'2", u'oid': {u'hash': 480221105, u'max': 0, u'oid': u'new_3', u'namespace': u'', u'snapid': -2, u'key': u'', u'pool': 1}, u'locations': [u'2(3)'], u'have': u"0'0"}], u'num_missing': 2, u'offset': {u'hash': 0, u'max': 0, u'oid': u'', u'namespace': u'', u'snapid': 0, u'key': u'', u'pool': -1}} 2014-09-25 21:49:26,990.990 INFO:tasks.ec_lost_unfound:reverting unfound in 1.1 2014-09-25 21:49:26,990.990 INFO:teuthology.orchestra.run.vpm057:Running: u'adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph pg 1.1 mark_unfound_lost delete' 2014-09-25 21:49:27,534.534 INFO:teuthology.orchestra.run.vpm057.stderr:pg has 2 objects unfound and apparently lost, marking 2014-09-25 21:49:27,545.545 INFO:teuthology.orchestra.run.vpm057:Running: 'adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph tell osd.0 debug kick_recovery_wq 5' 2014-09-25 21:49:28,112.112 INFO:teuthology.orchestra.run.vpm057.stderr:kicking recovery queue. set osd_recovery_delay_start to 5 2014-09-25 21:49:28,124.124 INFO:teuthology.orchestra.run.vpm057:Running: 'adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph tell osd.2 debug kick_recovery_wq 5' 2014-09-25 21:49:28,668.668 INFO:teuthology.orchestra.run.vpm057.stderr:kicking recovery queue. set osd_recovery_delay_start to 5 2014-09-25 21:49:28,684.684 INFO:teuthology.orchestra.run.vpm057:Running: 'adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph tell osd.3 debug kick_recovery_wq 5' 2014-09-25 21:49:29,341.341 INFO:teuthology.orchestra.run.vpm057.stderr:kicking recovery queue. set osd_recovery_delay_start to 5 2014-09-25 21:49:29,342.342 INFO:teuthology.orchestra.run.vpm057:Running: 'adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph tell osd.0 flush_pg_stats' 2014-09-25 21:49:30,152.152 INFO:teuthology.orchestra.run.vpm057:Running: 'adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph tell osd.2 flush_pg_stats' 2014-09-25 21:49:30,908.908 INFO:teuthology.orchestra.run.vpm057:Running: 'adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph tell osd.3 flush_pg_stats' 2014-09-25 21:49:31,708.708 INFO:tasks.ec_lost_unfound.ceph_manager:waiting for recovery to complete 2014-09-25 21:49:31,709.709 INFO:teuthology.orchestra.run.vpm057:Running: 'adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph pg dump --format=json' 2014-09-25 21:49:32,418.418 INFO:teuthology.orchestra.run.vpm057.stderr:dumped all in format json 2014-09-25 21:49:32,924.924 INFO:teuthology.orchestra.run.vpm057:Running: 'adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph pg dump --format=json' 2014-09-25 21:49:33,461.461 INFO:teuthology.orchestra.run.vpm057.stderr:dumped all in format json 2014-09-25 21:49:33,933.933 INFO:teuthology.orchestra.run.vpm057:Running: 'adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph status --format=json-pretty' 2014-09-25 21:49:34,604.604 INFO:tasks.ec_lost_unfound.ceph_manager:{u'election_epoch': 4, u'quorum': [0, 1, 2], u'mdsmap': {u'max': 0, u'epoch': 1, u'by_rank': [], u'up': 0, u'in': 0}, u'monmap': {u'epoch': 1, u'mons': [{u'name': u'a', u'rank': 0, u'addr': u'10.214.138.97:6789/0'}, {u'name': u'b', u'rank': 1, u'addr': u'10.214.138.97:6790/0'}, {u'name': u'c', u'rank': 2, u'addr': u'10.214.138.97:6791/0'}], u'modified': u'2014-09-25 19:46:21.446598', u'fsid': u'a0af5d53-05bd-40c8-9d30-7782574e4764', u'created': u'2014-09-25 19:46:21.446598'}, u'health': {u'detail': [], u'timechecks': {u'round_status': u'finished', u'epoch': 4, u'round': 2, u'mons': [{u'latency': u'0.000000', u'skew': u'0.000000', u'health': u'HEALTH_OK', u'name': u'a'}, {u'latency': u'0.003394', u'skew': u'0.000000', u'health': u'HEALTH_OK', u'name': u'b'}, {u'latency': u'0.018347', u'skew': u'0.000000', u'health': u'HEALTH_OK', u'name': u'c'}]}, u'health': {u'health_services': [{u'mons': [{u'last_updated': u'2014-09-25 19:49:02.482739', u'name': u'a', u'avail_percent': 90, u'kb_total': 103178528, u'kb_avail': 93892436, u'health': u'HEALTH_OK', u'kb_used': 5061132, u'store_stats': {u'bytes_total': 2051962, u'bytes_log': 2046359, u'last_updated': u'0.000000', u'bytes_misc': 5603, u'bytes_sst': 0}}, {u'last_updated': u'2014-09-25 19:49:02.838913', u'name': u'b', u'avail_percent': 90, u'kb_total': 103178528, u'kb_avail': 93892288, u'health': u'HEALTH_OK', u'kb_used': 5061280, u'store_stats': {u'bytes_total': 3074979, u'bytes_log': 3069376, u'last_updated': u'0.000000', u'bytes_misc': 5603, u'bytes_sst': 0}}, {u'last_updated': u'2014-09-25 19:49:03.251865', u'name': u'c', u'avail_percent': 90, u'kb_total': 103178528, u'kb_avail': 93892280, u'health': u'HEALTH_OK', u'kb_used': 5061288, u'store_stats': {u'bytes_total': 3074499, u'bytes_log': 3068896, u'last_updated': u'0.000000', u'bytes_misc': 5603, u'bytes_sst': 0}}]}]}, u'overall_status': u'HEALTH_WARN', u'summary': [{u'severity': u'HEALTH_WARN', u'summary': u'21 pgs degraded'}, {u'severity': u'HEALTH_WARN', u'summary': u'3 pgs stuck unclean'}, {u'severity': u'HEALTH_WARN', u'summary': u'7 pgs undersized'}, {u'severity': u'HEALTH_WARN', u'summary': u'recovery 11/112 objects degraded (9.821%); 28/112 objects misplaced (25.000%)'}, {u'severity': u'HEALTH_WARN', u'summary': u'1/4 in osds are down'}]}, u'pgmap': {u'bytes_total': 422619250688, u'degraded_objects': 11, u'num_pgs': 32, u'misplaced_objects': 28, u'data_bytes': 1204, u'degraded_total': 112, u'bytes_used': 20730548224, u'version': 57, u'pgs_by_state': [{u'count': 7, u'state_name': u'active+undersized+degraded'}, {u'count': 11, u'state_name': u'active+clean'}, {u'count': 14, u'state_name': u'active+clean+degraded'}], u'misplaced_ratio': u'25.000', u'degraded_ratio': u'9.821', u'bytes_avail': 384583266304, u'misplaced_total': 112}, u'quorum_names': [u'a', u'b', u'c'], u'osdmap': {u'osdmap': {u'full': False, u'nearfull': False, u'num_osds': 4, u'num_up_osds': 3, u'epoch': 20, u'num_in_osds': 4}}, u'fsid': u'a0af5d53-05bd-40c8-9d30-7782574e4764'} 2014-09-25 21:49:34,604.604 INFO:tasks.ec_lost_unfound.ceph_manager:recovered! 2014-09-25 21:49:34,605.605 INFO:tasks.util.rados:rados -p unique_pool_0 get new_1 - 2014-09-25 21:49:34,605.605 INFO:teuthology.orchestra.run.vpm057:Running: 'adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage rados -p unique_pool_0 get new_1 -' 2014-09-25 21:49:35,228.228 INFO:teuthology.orchestra.run.vpm057.stderr:error getting unique_pool_0/new_1: (2) No such file or directory 2014-09-25 21:49:35,230.230 INFO:tasks.util.rados:rados -p unique_pool_0 get existed_1 - 2014-09-25 21:49:35,231.231 INFO:teuthology.orchestra.run.vpm057:Running: 'adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage rados -p unique_pool_0 get existed_1 -' 2014-09-25 21:49:35,756.756 INFO:teuthology.orchestra.run.vpm057.stderr:error getting unique_pool_0/existed_1: (2) No such file or directory 2014-09-25 21:49:35,758.758 INFO:tasks.util.rados:rados -p unique_pool_0 get existing_1 - 2014-09-25 21:49:35,758.758 INFO:teuthology.orchestra.run.vpm057:Running: 'adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage rados -p unique_pool_0 get existing_1 -' 2014-09-25 21:49:36,567.567 INFO:teuthology.orchestra.run.vpm057.stderr:error getting unique_pool_0/existing_1: (2) No such file or directory 2014-09-25 21:49:36,569.569 INFO:tasks.util.rados:rados -p unique_pool_0 get new_2 - 2014-09-25 21:49:36,569.569 INFO:teuthology.orchestra.run.vpm057:Running: 'adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage rados -p unique_pool_0 get new_2 -' 2014-09-25 21:49:37,097.097 INFO:teuthology.orchestra.run.vpm057.stderr:error getting unique_pool_0/new_2: (2) No such file or directory 2014-09-25 21:49:37,099.099 INFO:tasks.util.rados:rados -p unique_pool_0 get existed_2 - 2014-09-25 21:49:37,099.099 INFO:teuthology.orchestra.run.vpm057:Running: 'adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage rados -p unique_pool_0 get existed_2 -' 2014-09-25 21:49:37,605.605 INFO:teuthology.orchestra.run.vpm057.stderr:error getting unique_pool_0/existed_2: (2) No such file or directory 2014-09-25 21:49:37,607.607 INFO:tasks.util.rados:rados -p unique_pool_0 get existing_2 - 2014-09-25 21:49:37,607.607 INFO:teuthology.orchestra.run.vpm057:Running: 'adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage rados -p unique_pool_0 get existing_2 -' 2014-09-25 21:49:38,132.132 INFO:teuthology.orchestra.run.vpm057.stderr:error getting unique_pool_0/existing_2: (2) No such file or directory 2014-09-25 21:49:38,133.133 INFO:tasks.util.rados:rados -p unique_pool_0 get new_3 - 2014-09-25 21:49:38,133.133 INFO:teuthology.orchestra.run.vpm057:Running: 'adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage rados -p unique_pool_0 get new_3 -' 2014-09-25 21:49:38,659.659 INFO:teuthology.orchestra.run.vpm057.stderr:error getting unique_pool_0/new_3: (2) No such file or directory 2014-09-25 21:49:38,660.660 INFO:tasks.util.rados:rados -p unique_pool_0 get existed_3 - 2014-09-25 21:49:38,660.660 INFO:teuthology.orchestra.run.vpm057:Running: 'adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage rados -p unique_pool_0 get existed_3 -' 2014-09-25 21:49:39,452.452 INFO:teuthology.orchestra.run.vpm057.stderr:error getting unique_pool_0/existed_3: (2) No such file or directory 2014-09-25 21:49:39,453.453 INFO:tasks.util.rados:rados -p unique_pool_0 get existing_3 - 2014-09-25 21:49:39,454.454 INFO:teuthology.orchestra.run.vpm057:Running: 'adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage rados -p unique_pool_0 get existing_3 -' 2014-09-25 21:49:39,978.978 INFO:teuthology.orchestra.run.vpm057.stderr:error getting unique_pool_0/existing_3: (2) No such file or directory 2014-09-25 21:49:39,981.981 INFO:tasks.util.rados:rados -p unique_pool_0 get new_4 - 2014-09-25 21:49:39,981.981 INFO:teuthology.orchestra.run.vpm057:Running: 'adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage rados -p unique_pool_0 get new_4 -' 2014-09-25 21:49:40,605.605 INFO:teuthology.orchestra.run.vpm057.stderr:error getting unique_pool_0/new_4: (2) No such file or directory 2014-09-25 21:49:40,608.608 INFO:tasks.util.rados:rados -p unique_pool_0 get existed_4 - 2014-09-25 21:49:40,609.609 INFO:teuthology.orchestra.run.vpm057:Running: 'adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage rados -p unique_pool_0 get existed_4 -' 2014-09-25 21:49:41,443.443 INFO:teuthology.orchestra.run.vpm057.stderr:error getting unique_pool_0/existed_4: (2) No such file or directory 2014-09-25 21:49:41,444.444 INFO:tasks.util.rados:rados -p unique_pool_0 get existing_4 - 2014-09-25 21:49:41,444.444 INFO:teuthology.orchestra.run.vpm057:Running: 'adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage rados -p unique_pool_0 get existing_4 -' 2014-09-25 21:49:42,004.004 INFO:teuthology.orchestra.run.vpm057.stderr:error getting unique_pool_0/existing_4: (2) No such file or directory 2014-09-25 21:49:42,006.006 INFO:tasks.util.rados:rados -p unique_pool_0 get new_5 - 2014-09-25 21:49:42,007.007 INFO:teuthology.orchestra.run.vpm057:Running: 'adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage rados -p unique_pool_0 get new_5 -' 2014-09-25 21:49:42,535.535 INFO:teuthology.orchestra.run.vpm057.stderr:error getting unique_pool_0/new_5: (2) No such file or directory 2014-09-25 21:49:42,536.536 INFO:tasks.util.rados:rados -p unique_pool_0 get existed_5 - 2014-09-25 21:49:42,536.536 INFO:teuthology.orchestra.run.vpm057:Running: 'adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage rados -p unique_pool_0 get existed_5 -' 2014-09-25 21:49:43,154.154 INFO:teuthology.orchestra.run.vpm057.stderr:error getting unique_pool_0/existed_5: (2) No such file or directory 2014-09-25 21:49:43,154.154 INFO:tasks.util.rados:rados -p unique_pool_0 get existing_5 - 2014-09-25 21:49:43,155.155 INFO:teuthology.orchestra.run.vpm057:Running: 'adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage rados -p unique_pool_0 get existing_5 -' 2014-09-25 21:49:43,800.800 INFO:teuthology.orchestra.run.vpm057.stderr:error getting unique_pool_0/existing_5: (2) No such file or directory 2014-09-25 21:49:43,800.800 INFO:tasks.util.rados:rados -p unique_pool_0 get new_6 - 2014-09-25 21:49:43,801.801 INFO:teuthology.orchestra.run.vpm057:Running: 'adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage rados -p unique_pool_0 get new_6 -' 2014-09-25 21:49:44,458.458 INFO:teuthology.orchestra.run.vpm057.stderr:error getting unique_pool_0/new_6: (2) No such file or directory 2014-09-25 21:49:44,459.459 INFO:tasks.util.rados:rados -p unique_pool_0 get existed_6 - 2014-09-25 21:49:44,460.460 INFO:teuthology.orchestra.run.vpm057:Running: 'adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage rados -p unique_pool_0 get existed_6 -' 2014-09-25 21:49:45,152.152 INFO:teuthology.orchestra.run.vpm057.stderr:error getting unique_pool_0/existed_6: (2) No such file or directory 2014-09-25 21:49:45,152.152 INFO:tasks.util.rados:rados -p unique_pool_0 get existing_6 - 2014-09-25 21:49:45,153.153 INFO:teuthology.orchestra.run.vpm057:Running: 'adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage rados -p unique_pool_0 get existing_6 -' 2014-09-25 21:49:45,948.948 INFO:teuthology.orchestra.run.vpm057.stderr:error getting unique_pool_0/existing_6: (2) No such file or directory 2014-09-25 21:49:45,949.949 INFO:tasks.util.rados:rados -p unique_pool_0 get new_7 - 2014-09-25 21:49:45,949.949 INFO:teuthology.orchestra.run.vpm057:Running: 'adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage rados -p unique_pool_0 get new_7 -' 2014-09-25 21:49:46,655.655 INFO:teuthology.orchestra.run.vpm057.stderr:error getting unique_pool_0/new_7: (2) No such file or directory 2014-09-25 21:49:46,655.655 INFO:tasks.util.rados:rados -p unique_pool_0 get existed_7 - 2014-09-25 21:49:46,656.656 INFO:teuthology.orchestra.run.vpm057:Running: 'adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage rados -p unique_pool_0 get existed_7 -' 2014-09-25 21:49:47,215.215 INFO:teuthology.orchestra.run.vpm057.stderr:error getting unique_pool_0/existed_7: (2) No such file or directory 2014-09-25 21:49:47,216.216 INFO:tasks.util.rados:rados -p unique_pool_0 get existing_7 - 2014-09-25 21:49:47,216.216 INFO:teuthology.orchestra.run.vpm057:Running: 'adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage rados -p unique_pool_0 get existing_7 -' 2014-09-25 21:49:47,792.792 INFO:teuthology.orchestra.run.vpm057.stderr:error getting unique_pool_0/existing_7: (2) No such file or directory 2014-09-25 21:49:47,792.792 INFO:tasks.util.rados:rados -p unique_pool_0 get new_8 - 2014-09-25 21:49:47,792.792 INFO:teuthology.orchestra.run.vpm057:Running: 'adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage rados -p unique_pool_0 get new_8 -' 2014-09-25 21:49:48,407.407 INFO:teuthology.orchestra.run.vpm057.stderr:error getting unique_pool_0/new_8: (2) No such file or directory 2014-09-25 21:49:48,407.407 INFO:tasks.util.rados:rados -p unique_pool_0 get existed_8 - 2014-09-25 21:49:48,407.407 INFO:teuthology.orchestra.run.vpm057:Running: 'adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage rados -p unique_pool_0 get existed_8 -' 2014-09-25 21:49:49,125.125 INFO:teuthology.orchestra.run.vpm057.stderr:error getting unique_pool_0/existed_8: (2) No such file or directory 2014-09-25 21:49:49,126.126 INFO:tasks.util.rados:rados -p unique_pool_0 get existing_8 - 2014-09-25 21:49:49,126.126 INFO:teuthology.orchestra.run.vpm057:Running: 'adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage rados -p unique_pool_0 get existing_8 -' 2014-09-25 21:49:49,842.842 INFO:teuthology.orchestra.run.vpm057.stderr:error getting unique_pool_0/existing_8: (2) No such file or directory 2014-09-25 21:49:49,843.843 INFO:tasks.util.rados:rados -p unique_pool_0 get new_9 - 2014-09-25 21:49:49,843.843 INFO:teuthology.orchestra.run.vpm057:Running: 'adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage rados -p unique_pool_0 get new_9 -' 2014-09-25 21:49:50,557.557 INFO:teuthology.orchestra.run.vpm057.stderr:error getting unique_pool_0/new_9: (2) No such file or directory 2014-09-25 21:49:50,558.558 INFO:tasks.util.rados:rados -p unique_pool_0 get existed_9 - 2014-09-25 21:49:50,558.558 INFO:teuthology.orchestra.run.vpm057:Running: 'adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage rados -p unique_pool_0 get existed_9 -' 2014-09-25 21:49:51,353.353 INFO:teuthology.orchestra.run.vpm057.stderr:error getting unique_pool_0/existed_9: (2) No such file or directory 2014-09-25 21:49:51,355.355 INFO:tasks.util.rados:rados -p unique_pool_0 get existing_9 - 2014-09-25 21:49:51,355.355 INFO:teuthology.orchestra.run.vpm057:Running: 'adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage rados -p unique_pool_0 get existing_9 -' 2014-09-25 21:49:51,908.908 INFO:teuthology.orchestra.run.vpm057.stderr:error getting unique_pool_0/existing_9: (2) No such file or directory 2014-09-25 21:49:51,909.909 INFO:tasks.ceph.osd.1:Restarting daemon 2014-09-25 21:49:51,910.910 INFO:teuthology.orchestra.run.vpm057:Running: 'sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage daemon-helper kill ceph-osd -f -i 1' 2014-09-25 21:49:52,502.502 INFO:tasks.ceph.osd.1:Started 2014-09-25 21:49:52,503.503 INFO:teuthology.orchestra.run.vpm057:Running: 'sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph --admin-daemon /var/run/ceph/ceph-osd.1.asok dump_ops_in_flight' 2014-09-25 21:49:52,504.504 INFO:tasks.ceph.osd.1.vpm057.stdout:starting osd.1 at :/0 osd_data /var/lib/ceph/osd/ceph-1 /var/lib/ceph/osd/ceph-1/journal 2014-09-25 21:49:52,505.505 INFO:tasks.ceph.osd.1.vpm057.stderr:2014-09-25 19:49:52.418623 7facef2d1900 -1 journal FileJournal::_open: disabling aio for non-block journal. Use journal_force_aio to force use of aio anyway 2014-09-25 21:49:53,014.014 INFO:tasks.ec_lost_unfound.ceph_manager:waiting for osd.1 to be up 2014-09-25 21:49:53,015.015 INFO:teuthology.orchestra.run.vpm057:Running: 'adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph osd dump --format=json' 2014-09-25 21:49:53,632.632 INFO:tasks.ec_lost_unfound.ceph_manager:osd.1 is up 2014-09-25 21:49:53,633.633 INFO:tasks.ec_lost_unfound.ceph_manager:waiting for clean 2014-09-25 21:49:53,633.633 INFO:teuthology.orchestra.run.vpm057:Running: 'adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph pg dump --format=json' 2014-09-25 21:49:54,550.550 INFO:teuthology.orchestra.run.vpm057.stderr:dumped all in format json 2014-09-25 21:49:55,037.037 INFO:teuthology.orchestra.run.vpm057:Running: 'adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph pg dump --format=json' 2014-09-25 21:49:55,829.829 INFO:teuthology.orchestra.run.vpm057.stderr:dumped all in format json 2014-09-25 21:49:56,028.028 INFO:teuthology.orchestra.run.vpm057:Running: 'adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph status --format=json-pretty' 2014-09-25 21:49:56,757.757 INFO:tasks.ec_lost_unfound.ceph_manager:{u'election_epoch': 4, u'quorum': [0, 1, 2], u'mdsmap': {u'max': 0, u'epoch': 1, u'by_rank': [], u'up': 0, u'in': 0}, u'monmap': {u'epoch': 1, u'mons': [{u'name': u'a', u'rank': 0, u'addr': u'10.214.138.97:6789/0'}, {u'name': u'b', u'rank': 1, u'addr': u'10.214.138.97:6790/0'}, {u'name': u'c', u'rank': 2, u'addr': u'10.214.138.97:6791/0'}], u'modified': u'2014-09-25 19:46:21.446598', u'fsid': u'a0af5d53-05bd-40c8-9d30-7782574e4764', u'created': u'2014-09-25 19:46:21.446598'}, u'health': {u'detail': [], u'timechecks': {u'round_status': u'finished', u'epoch': 4, u'round': 2, u'mons': [{u'latency': u'0.000000', u'skew': u'0.000000', u'health': u'HEALTH_OK', u'name': u'a'}, {u'latency': u'0.003394', u'skew': u'0.000000', u'health': u'HEALTH_OK', u'name': u'b'}, {u'latency': u'0.018347', u'skew': u'0.000000', u'health': u'HEALTH_OK', u'name': u'c'}]}, u'health': {u'health_services': [{u'mons': [{u'last_updated': u'2014-09-25 19:49:02.482739', u'name': u'a', u'avail_percent': 90, u'kb_total': 103178528, u'kb_avail': 93892436, u'health': u'HEALTH_OK', u'kb_used': 5061132, u'store_stats': {u'bytes_total': 2051962, u'bytes_log': 2046359, u'last_updated': u'0.000000', u'bytes_misc': 5603, u'bytes_sst': 0}}, {u'last_updated': u'2014-09-25 19:49:02.838913', u'name': u'b', u'avail_percent': 90, u'kb_total': 103178528, u'kb_avail': 93892288, u'health': u'HEALTH_OK', u'kb_used': 5061280, u'store_stats': {u'bytes_total': 3074979, u'bytes_log': 3069376, u'last_updated': u'0.000000', u'bytes_misc': 5603, u'bytes_sst': 0}}, {u'last_updated': u'2014-09-25 19:49:03.251865', u'name': u'c', u'avail_percent': 90, u'kb_total': 103178528, u'kb_avail': 93892280, u'health': u'HEALTH_OK', u'kb_used': 5061288, u'store_stats': {u'bytes_total': 3074499, u'bytes_log': 3068896, u'last_updated': u'0.000000', u'bytes_misc': 5603, u'bytes_sst': 0}}]}]}, u'overall_status': u'HEALTH_WARN', u'summary': [{u'severity': u'HEALTH_WARN', u'summary': u'21 pgs degraded'}, {u'severity': u'HEALTH_WARN', u'summary': u'3 pgs stuck unclean'}, {u'severity': u'HEALTH_WARN', u'summary': u'7 pgs undersized'}, {u'severity': u'HEALTH_WARN', u'summary': u'recovery 11/112 objects degraded (9.821%); 28/112 objects misplaced (25.000%)'}]}, u'pgmap': {u'bytes_total': 422619250688, u'degraded_objects': 11, u'num_pgs': 32, u'misplaced_objects': 28, u'data_bytes': 1204, u'degraded_total': 112, u'bytes_used': 20730548224, u'version': 59, u'pgs_by_state': [{u'count': 7, u'state_name': u'active+undersized+degraded'}, {u'count': 11, u'state_name': u'active+clean'}, {u'count': 14, u'state_name': u'active+clean+degraded'}], u'misplaced_ratio': u'25.000', u'degraded_ratio': u'9.821', u'bytes_avail': 384583266304, u'misplaced_total': 112}, u'quorum_names': [u'a', u'b', u'c'], u'osdmap': {u'osdmap': {u'full': False, u'nearfull': False, u'num_osds': 4, u'num_up_osds': 4, u'epoch': 22, u'num_in_osds': 4}}, u'fsid': u'a0af5d53-05bd-40c8-9d30-7782574e4764'} 2014-09-25 21:49:56,757.757 INFO:teuthology.orchestra.run.vpm057:Running: 'adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph pg dump --format=json' 2014-09-25 21:49:57,482.482 INFO:teuthology.orchestra.run.vpm057.stderr:dumped all in format json 2014-09-25 21:50:00,683.683 INFO:teuthology.orchestra.run.vpm057:Running: 'adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph pg dump --format=json' 2014-09-25 21:50:01,404.404 INFO:teuthology.orchestra.run.vpm057.stderr:dumped all in format json 2014-09-25 21:50:01,862.862 INFO:teuthology.orchestra.run.vpm057:Running: 'adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph status --format=json-pretty' 2014-09-25 21:50:02,575.575 INFO:tasks.ec_lost_unfound.ceph_manager:{u'election_epoch': 4, u'quorum': [0, 1, 2], u'mdsmap': {u'max': 0, u'epoch': 1, u'by_rank': [], u'up': 0, u'in': 0}, u'monmap': {u'epoch': 1, u'mons': [{u'name': u'a', u'rank': 0, u'addr': u'10.214.138.97:6789/0'}, {u'name': u'b', u'rank': 1, u'addr': u'10.214.138.97:6790/0'}, {u'name': u'c', u'rank': 2, u'addr': u'10.214.138.97:6791/0'}], u'modified': u'2014-09-25 19:46:21.446598', u'fsid': u'a0af5d53-05bd-40c8-9d30-7782574e4764', u'created': u'2014-09-25 19:46:21.446598'}, u'health': {u'detail': [], u'timechecks': {u'round_status': u'finished', u'epoch': 4, u'round': 2, u'mons': [{u'latency': u'0.000000', u'skew': u'0.000000', u'health': u'HEALTH_OK', u'name': u'a'}, {u'latency': u'0.003394', u'skew': u'0.000000', u'health': u'HEALTH_OK', u'name': u'b'}, {u'latency': u'0.018347', u'skew': u'0.000000', u'health': u'HEALTH_OK', u'name': u'c'}]}, u'health': {u'health_services': [{u'mons': [{u'last_updated': u'2014-09-25 19:49:02.482739', u'name': u'a', u'avail_percent': 90, u'kb_total': 103178528, u'kb_avail': 93892436, u'health': u'HEALTH_OK', u'kb_used': 5061132, u'store_stats': {u'bytes_total': 2051962, u'bytes_log': 2046359, u'last_updated': u'0.000000', u'bytes_misc': 5603, u'bytes_sst': 0}}, {u'last_updated': u'2014-09-25 19:49:02.838913', u'name': u'b', u'avail_percent': 90, u'kb_total': 103178528, u'kb_avail': 93892288, u'health': u'HEALTH_OK', u'kb_used': 5061280, u'store_stats': {u'bytes_total': 3074979, u'bytes_log': 3069376, u'last_updated': u'0.000000', u'bytes_misc': 5603, u'bytes_sst': 0}}, {u'last_updated': u'2014-09-25 19:49:03.251865', u'name': u'c', u'avail_percent': 90, u'kb_total': 103178528, u'kb_avail': 93892280, u'health': u'HEALTH_OK', u'kb_used': 5061288, u'store_stats': {u'bytes_total': 3074499, u'bytes_log': 3068896, u'last_updated': u'0.000000', u'bytes_misc': 5603, u'bytes_sst': 0}}]}]}, u'overall_status': u'HEALTH_OK', u'summary': []}, u'pgmap': {u'bytes_total': 422619250688, u'num_pgs': 32, u'data_bytes': 774, u'bytes_used': 20743487488, u'version': 61, u'pgs_by_state': [{u'count': 32, u'state_name': u'active+clean'}], u'bytes_avail': 384570327040}, u'quorum_names': [u'a', u'b', u'c'], u'osdmap': {u'osdmap': {u'full': False, u'nearfull': False, u'num_osds': 4, u'num_up_osds': 4, u'epoch': 22, u'num_in_osds': 4}}, u'fsid': u'a0af5d53-05bd-40c8-9d30-7782574e4764'} 2014-09-25 21:50:02,575.575 INFO:tasks.ec_lost_unfound.ceph_manager:clean! 2014-09-25 21:50:02,575.575 DEBUG:teuthology.run_tasks:Unwinding manager ceph 2014-09-25 21:50:02,576.576 INFO:teuthology.orchestra.run.vpm057:Running: 'adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph pg dump --format json' 2014-09-25 21:50:03,458.458 INFO:teuthology.orchestra.run.vpm057.stderr:dumped all in format json 2014-09-25 21:50:04,771.771 INFO:tasks.ceph:Scrubbing osd osd.0 2014-09-25 21:50:04,772.772 INFO:teuthology.orchestra.run.vpm057:Running: 'adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph osd deep-scrub osd.0' 2014-09-25 21:50:05,506.506 INFO:teuthology.orchestra.run.vpm057.stderr:osd.0 instructed to deep-scrub 2014-09-25 21:50:05,506.506 INFO:tasks.ceph:Scrubbing osd osd.1 2014-09-25 21:50:05,507.507 INFO:teuthology.orchestra.run.vpm057:Running: 'adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph osd deep-scrub osd.1' 2014-09-25 21:50:06,406.406 INFO:teuthology.orchestra.run.vpm057.stderr:osd.1 instructed to deep-scrub 2014-09-25 21:50:06,417.417 INFO:tasks.ceph:Scrubbing osd osd.2 2014-09-25 21:50:06,417.417 INFO:teuthology.orchestra.run.vpm057:Running: 'adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph osd deep-scrub osd.2' 2014-09-25 21:50:07,125.125 INFO:teuthology.orchestra.run.vpm057.stderr:osd.2 instructed to deep-scrub 2014-09-25 21:50:07,136.136 INFO:tasks.ceph:Scrubbing osd osd.3 2014-09-25 21:50:07,136.136 INFO:teuthology.orchestra.run.vpm057:Running: 'adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph osd deep-scrub osd.3' 2014-09-25 21:50:07,749.749 INFO:teuthology.orchestra.run.vpm057.stderr:osd.3 instructed to deep-scrub 2014-09-25 21:50:07,762.762 INFO:teuthology.orchestra.run.vpm057:Running: 'adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph pg dump --format json' 2014-09-25 21:50:08,544.544 INFO:teuthology.orchestra.run.vpm057.stderr:dumped all in format json 2014-09-25 21:50:09,187.187 INFO:tasks.ceph:Still waiting for all pgs to be scrubbed. 2014-09-25 21:50:19,174.174 INFO:teuthology.orchestra.run.vpm057:Running: 'adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph pg dump --format json' 2014-09-25 21:50:19,754.754 INFO:teuthology.orchestra.run.vpm057.stderr:dumped all in format json 2014-09-25 21:50:20,092.092 INFO:tasks.ceph:Still waiting for all pgs to be scrubbed. 2014-09-25 21:50:30,089.089 INFO:teuthology.orchestra.run.vpm057:Running: 'adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph pg dump --format json' 2014-09-25 21:50:30,651.651 INFO:teuthology.orchestra.run.vpm057.stderr:dumped all in format json 2014-09-25 21:50:31,023.023 INFO:tasks.ceph:Still waiting for all pgs to be scrubbed. 2014-09-25 21:50:41,019.019 INFO:teuthology.orchestra.run.vpm057:Running: 'adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph pg dump --format json' 2014-09-25 21:50:41,654.654 INFO:teuthology.orchestra.run.vpm057.stderr:dumped all in format json 2014-09-25 21:50:42,004.004 INFO:tasks.ceph:Still waiting for all pgs to be scrubbed. 2014-09-25 21:50:51,997.997 INFO:teuthology.orchestra.run.vpm057:Running: 'adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph pg dump --format json' 2014-09-25 21:50:52,534.534 INFO:teuthology.orchestra.run.vpm057.stderr:dumped all in format json 2014-09-25 21:50:52,954.954 INFO:tasks.ceph:Still waiting for all pgs to be scrubbed. 2014-09-25 21:51:02,948.948 INFO:teuthology.orchestra.run.vpm057:Running: 'adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph pg dump --format json' 2014-09-25 21:51:03,531.531 INFO:teuthology.orchestra.run.vpm057.stderr:dumped all in format json 2014-09-25 21:51:03,960.960 INFO:tasks.ceph:Still waiting for all pgs to be scrubbed. 2014-09-25 21:51:13,954.954 INFO:teuthology.orchestra.run.vpm057:Running: 'adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph pg dump --format json' 2014-09-25 21:51:14,628.628 INFO:teuthology.orchestra.run.vpm057.stderr:dumped all in format json 2014-09-25 21:51:14,972.972 INFO:tasks.ceph:Still waiting for all pgs to be scrubbed. 2014-09-25 21:51:24,964.964 INFO:teuthology.orchestra.run.vpm057:Running: 'adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph pg dump --format json' 2014-09-25 21:51:25,515.515 INFO:teuthology.orchestra.run.vpm057.stderr:dumped all in format json 2014-09-25 21:51:25,935.935 INFO:tasks.ceph:Still waiting for all pgs to be scrubbed. 2014-09-25 21:51:35,932.932 INFO:teuthology.orchestra.run.vpm057:Running: 'adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph pg dump --format json' 2014-09-25 21:51:36,478.478 INFO:teuthology.orchestra.run.vpm057.stderr:dumped all in format json 2014-09-25 21:51:36,823.823 INFO:tasks.ceph:Still waiting for all pgs to be scrubbed. 2014-09-25 21:51:46,815.815 INFO:teuthology.orchestra.run.vpm057:Running: 'adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph pg dump --format json' 2014-09-25 21:51:47,325.325 INFO:teuthology.orchestra.run.vpm057.stderr:dumped all in format json 2014-09-25 21:51:47,666.666 INFO:tasks.ceph:Still waiting for all pgs to be scrubbed. 2014-09-25 21:51:57,659.659 INFO:teuthology.orchestra.run.vpm057:Running: 'adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph pg dump --format json' 2014-09-25 21:51:58,155.155 INFO:teuthology.orchestra.run.vpm057.stderr:dumped all in format json 2014-09-25 21:51:58,492.492 INFO:tasks.ceph:Still waiting for all pgs to be scrubbed. 2014-09-25 21:52:08,489.489 INFO:teuthology.orchestra.run.vpm057:Running: 'adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph pg dump --format json' 2014-09-25 21:52:09,141.141 INFO:teuthology.orchestra.run.vpm057.stderr:dumped all in format json 2014-09-25 21:52:09,479.479 INFO:tasks.ceph:Still waiting for all pgs to be scrubbed. 2014-09-25 21:52:19,471.471 INFO:teuthology.orchestra.run.vpm057:Running: 'adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph pg dump --format json' 2014-09-25 21:52:20,061.061 INFO:teuthology.orchestra.run.vpm057.stderr:dumped all in format json 2014-09-25 21:52:20,542.542 INFO:tasks.ceph:Exiting scrub checking -- not all pgs scrubbed. 2014-09-25 21:52:20,543.543 INFO:teuthology.misc:Shutting down mds daemons... 2014-09-25 21:52:20,543.543 INFO:teuthology.misc:Shutting down osd daemons... 2014-09-25 21:52:26,536.536 INFO:tasks.ceph.osd.1:Stopped 2014-09-25 21:52:32,537.537 INFO:tasks.ceph.osd.0:Stopped 2014-09-25 21:52:38,538.538 INFO:tasks.ceph.osd.3:Stopped 2014-09-25 21:52:44,539.539 INFO:tasks.ceph.osd.2:Stopped 2014-09-25 21:52:44,540.540 INFO:teuthology.misc:Shutting down mon daemons... 2014-09-25 21:52:50,540.540 INFO:tasks.ceph.mon.a:Stopped 2014-09-25 21:52:56,541.541 INFO:tasks.ceph.mon.c:Stopped 2014-09-25 21:53:02,543.543 INFO:tasks.ceph.mon.b:Stopped 2014-09-25 21:53:02,543.543 INFO:tasks.ceph:Checking cluster log for badness... 2014-09-25 21:53:02,544.544 INFO:teuthology.orchestra.run.vpm057:Running: "sudo egrep '\\[ERR\\]|\\[WRN\\]|\\[SEC\\]' /var/log/ceph/ceph.log | egrep -v 'objects unfound and apparently lost' | head -n 1" 2014-09-25 21:53:03,011.011 INFO:tasks.ceph:Cleaning ceph cluster... 2014-09-25 21:53:03,011.011 INFO:teuthology.orchestra.run.vpm057:Running: 'sudo rm -rf -- /etc/ceph/ceph.conf /etc/ceph/ceph.keyring /home/ubuntu/cephtest/data /home/ubuntu/cephtest/monmap' 2014-09-25 21:53:03,638.638 INFO:tasks.ceph:Checking for errors in any valgrind logs... 2014-09-25 21:53:03,638.638 INFO:teuthology.orchestra.run.vpm057:Running: "sudo zgrep '' /var/log/ceph/valgrind/* /dev/null | sort | uniq" 2014-09-25 21:53:05,882.882 INFO:teuthology.orchestra.run.vpm057.stderr:gzip: /var/log/ceph/valgrind/*.gz: No such file or directory 2014-09-25 21:53:05,884.884 DEBUG:teuthology.run_tasks:Unwinding manager install 2014-09-25 21:53:05,884.884 INFO:teuthology.task.install:Removing shipped files: /home/ubuntu/cephtest/valgrind.supp /usr/bin/daemon-helper /usr/bin/adjust-ulimits... 2014-09-25 21:53:05,884.884 INFO:teuthology.orchestra.run.vpm057:Running: 'sudo rm -f -- /home/ubuntu/cephtest/valgrind.supp /usr/bin/daemon-helper /usr/bin/adjust-ulimits' 2014-09-25 21:53:08,168.168 INFO:teuthology.orchestra.run.vpm057:Running: 'sudo lsb_release -is' 2014-09-25 21:53:08,787.787 DEBUG:teuthology.misc:System to be installed: Ubuntu 2014-09-25 21:53:08,787.787 INFO:teuthology.task.install:Removing packages: ceph, ceph-dbg, ceph-mds, ceph-mds-dbg, ceph-common, ceph-common-dbg, ceph-fuse, ceph-fuse-dbg, ceph-test, ceph-test-dbg, radosgw, radosgw-dbg, python-ceph, libcephfs1, libcephfs1-dbg, libcephfs-java on Debian system. 2014-09-25 21:53:08,788.788 INFO:teuthology.orchestra.run.vpm057:Running: 'for d in ceph ceph-dbg ceph-mds ceph-mds-dbg ceph-common ceph-common-dbg ceph-fuse ceph-fuse-dbg ceph-test ceph-test-dbg radosgw radosgw-dbg python-ceph libcephfs1 libcephfs1-dbg libcephfs-java ; do sudo DEBIAN_FRONTEND=noninteractive apt-get -y --force-yes -o Dpkg::Options::="--force-confdef" -o Dpkg::Options::="--force-confold" purge $d || true ; done' 2014-09-25 21:53:09,437.437 INFO:teuthology.orchestra.run.vpm057.stdout:Reading package lists... 2014-09-25 21:53:09,602.602 INFO:teuthology.orchestra.run.vpm057.stdout:Building dependency tree... 2014-09-25 21:53:09,603.603 INFO:teuthology.orchestra.run.vpm057.stdout:Reading state information... 2014-09-25 21:53:09,751.751 INFO:teuthology.orchestra.run.vpm057.stdout:The following packages will be REMOVED: 2014-09-25 21:53:09,752.752 INFO:teuthology.orchestra.run.vpm057.stdout: ceph* ceph-dbg* ceph-mds* ceph-mds-dbg* 2014-09-25 21:53:09,868.868 INFO:teuthology.orchestra.run.vpm057.stdout:0 upgraded, 0 newly installed, 4 to remove and 207 not upgraded. 2014-09-25 21:53:09,868.868 INFO:teuthology.orchestra.run.vpm057.stdout:After this operation, 797 MB disk space will be freed. 2014-09-25 21:53:09,957.957 INFO:teuthology.orchestra.run.vpm057.stdout:(Reading database ... 53053 files and directories currently installed.) 2014-09-25 21:53:09,957.957 INFO:teuthology.orchestra.run.vpm057.stdout:Removing ceph-mds-dbg (0.85-988-g7552571-1trusty) ... 2014-09-25 21:53:09,980.980 INFO:teuthology.orchestra.run.vpm057.stdout:Removing ceph-mds (0.85-988-g7552571-1trusty) ... 2014-09-25 21:53:09,988.988 INFO:teuthology.orchestra.run.vpm057.stdout:ceph-mds-all stop/waiting 2014-09-25 21:53:10,103.103 INFO:teuthology.orchestra.run.vpm057.stdout:/etc/init.d/ceph: ceph conf /etc/ceph/ceph.conf not found; system is not configured. 2014-09-25 21:53:10,137.137 INFO:teuthology.orchestra.run.vpm057.stdout:Purging configuration files for ceph-mds (0.85-988-g7552571-1trusty) ... 2014-09-25 21:53:10,173.173 INFO:teuthology.orchestra.run.vpm057.stdout:Removing ceph-dbg (0.85-988-g7552571-1trusty) ... 2014-09-25 21:53:10,225.225 INFO:teuthology.orchestra.run.vpm057.stdout:Removing ceph (0.85-988-g7552571-1trusty) ... 2014-09-25 21:53:10,257.257 INFO:teuthology.orchestra.run.vpm057.stdout:ceph-all stop/waiting 2014-09-25 21:53:10,272.272 INFO:teuthology.orchestra.run.vpm057.stdout:/etc/init.d/ceph: ceph conf /etc/ceph/ceph.conf not found; system is not configured. 2014-09-25 21:53:10,302.302 INFO:teuthology.orchestra.run.vpm057.stdout:Purging configuration files for ceph (0.85-988-g7552571-1trusty) ... 2014-09-25 21:53:10,355.355 INFO:teuthology.orchestra.run.vpm057.stderr:dpkg: warning: while removing ceph, directory '/var/lib/ceph/osd' not empty so not removed 2014-09-25 21:53:10,356.356 INFO:teuthology.orchestra.run.vpm057.stderr:dpkg: warning: while removing ceph, directory '/var/lib/ceph/mon' not empty so not removed 2014-09-25 21:53:10,375.375 INFO:teuthology.orchestra.run.vpm057.stdout:Processing triggers for man-db (2.6.6-1) ... 2014-09-25 21:53:10,613.613 INFO:teuthology.orchestra.run.vpm057.stdout:Processing triggers for libc-bin (2.19-0ubuntu2) ... 2014-09-25 21:53:11,521.521 INFO:teuthology.orchestra.run.vpm057.stdout:Reading package lists... 2014-09-25 21:53:11,641.641 INFO:teuthology.orchestra.run.vpm057.stdout:Building dependency tree... 2014-09-25 21:53:11,643.643 INFO:teuthology.orchestra.run.vpm057.stdout:Reading state information... 2014-09-25 21:53:11,711.711 INFO:teuthology.orchestra.run.vpm057.stdout:Package 'ceph-dbg' is not installed, so not removed 2014-09-25 21:53:11,711.711 INFO:teuthology.orchestra.run.vpm057.stdout:The following packages were automatically installed and are no longer required: 2014-09-25 21:53:11,713.713 INFO:teuthology.orchestra.run.vpm057.stdout: btrfs-tools ceph-fs-common liblzo2-2 libreadline5 xfsprogs 2014-09-25 21:53:11,714.714 INFO:teuthology.orchestra.run.vpm057.stdout:Use 'apt-get autoremove' to remove them. 2014-09-25 21:53:11,728.728 INFO:teuthology.orchestra.run.vpm057.stdout:0 upgraded, 0 newly installed, 0 to remove and 207 not upgraded. 2014-09-25 21:53:11,737.737 INFO:teuthology.orchestra.run.vpm057.stdout:Reading package lists... 2014-09-25 21:53:11,889.889 INFO:teuthology.orchestra.run.vpm057.stdout:Building dependency tree... 2014-09-25 21:53:11,889.889 INFO:teuthology.orchestra.run.vpm057.stdout:Reading state information... 2014-09-25 21:53:12,003.003 INFO:teuthology.orchestra.run.vpm057.stdout:Package 'ceph-mds' is not installed, so not removed 2014-09-25 21:53:12,003.003 INFO:teuthology.orchestra.run.vpm057.stdout:The following packages were automatically installed and are no longer required: 2014-09-25 21:53:12,004.004 INFO:teuthology.orchestra.run.vpm057.stdout: btrfs-tools ceph-fs-common liblzo2-2 libreadline5 xfsprogs 2014-09-25 21:53:12,005.005 INFO:teuthology.orchestra.run.vpm057.stdout:Use 'apt-get autoremove' to remove them. 2014-09-25 21:53:12,024.024 INFO:teuthology.orchestra.run.vpm057.stdout:0 upgraded, 0 newly installed, 0 to remove and 207 not upgraded. 2014-09-25 21:53:12,033.033 INFO:teuthology.orchestra.run.vpm057.stdout:Reading package lists... 2014-09-25 21:53:12,240.240 INFO:teuthology.orchestra.run.vpm057.stdout:Building dependency tree... 2014-09-25 21:53:12,240.240 INFO:teuthology.orchestra.run.vpm057.stdout:Reading state information... 2014-09-25 21:53:12,297.297 INFO:teuthology.orchestra.run.vpm057.stdout:Package 'ceph-mds-dbg' is not installed, so not removed 2014-09-25 21:53:12,297.297 INFO:teuthology.orchestra.run.vpm057.stdout:The following packages were automatically installed and are no longer required: 2014-09-25 21:53:12,297.297 INFO:teuthology.orchestra.run.vpm057.stdout: btrfs-tools ceph-fs-common liblzo2-2 libreadline5 xfsprogs 2014-09-25 21:53:12,298.298 INFO:teuthology.orchestra.run.vpm057.stdout:Use 'apt-get autoremove' to remove them. 2014-09-25 21:53:12,323.323 INFO:teuthology.orchestra.run.vpm057.stdout:0 upgraded, 0 newly installed, 0 to remove and 207 not upgraded. 2014-09-25 21:53:12,332.332 INFO:teuthology.orchestra.run.vpm057.stdout:Reading package lists... 2014-09-25 21:53:12,571.571 INFO:teuthology.orchestra.run.vpm057.stdout:Building dependency tree... 2014-09-25 21:53:12,572.572 INFO:teuthology.orchestra.run.vpm057.stdout:Reading state information... 2014-09-25 21:53:12,680.680 INFO:teuthology.orchestra.run.vpm057.stdout:The following packages were automatically installed and are no longer required: 2014-09-25 21:53:12,682.682 INFO:teuthology.orchestra.run.vpm057.stdout: btrfs-tools ceph-fs-common libaio1 libbabeltrace-ctf1 libbabeltrace1 2014-09-25 21:53:12,682.682 INFO:teuthology.orchestra.run.vpm057.stdout: libboost-program-options1.54.0 libfcgi0ldbl libgoogle-perftools4 libleveldb1 2014-09-25 21:53:12,682.682 INFO:teuthology.orchestra.run.vpm057.stdout: liblzo2-2 libradosstriper1 libreadline5 libsnappy1 libtcmalloc-minimal4 2014-09-25 21:53:12,683.683 INFO:teuthology.orchestra.run.vpm057.stdout: libunwind8 xfsprogs xml2 2014-09-25 21:53:12,684.684 INFO:teuthology.orchestra.run.vpm057.stdout:Use 'apt-get autoremove' to remove them. 2014-09-25 21:53:12,718.718 INFO:teuthology.orchestra.run.vpm057.stdout:The following packages will be REMOVED: 2014-09-25 21:53:12,719.719 INFO:teuthology.orchestra.run.vpm057.stdout: ceph-common* ceph-common-dbg* ceph-test* ceph-test-dbg* radosgw* 2014-09-25 21:53:12,719.719 INFO:teuthology.orchestra.run.vpm057.stdout: radosgw-dbg* 2014-09-25 21:53:12,864.864 INFO:teuthology.orchestra.run.vpm057.stdout:0 upgraded, 0 newly installed, 6 to remove and 207 not upgraded. 2014-09-25 21:53:12,864.864 INFO:teuthology.orchestra.run.vpm057.stdout:After this operation, 2,143 MB disk space will be freed. 2014-09-25 21:53:13,240.240 INFO:teuthology.orchestra.run.vpm057.stdout:(Reading database ... 52841 files and directories currently installed.) 2014-09-25 21:53:13,243.243 INFO:teuthology.orchestra.run.vpm057.stdout:Removing radosgw-dbg (0.85-988-g7552571-1trusty) ... 2014-09-25 21:53:13,310.310 INFO:teuthology.orchestra.run.vpm057.stdout:Removing radosgw (0.85-988-g7552571-1trusty) ... 2014-09-25 21:53:13,355.355 INFO:teuthology.orchestra.run.vpm057.stderr:stop: Unknown instance: 2014-09-25 21:53:13,668.668 INFO:teuthology.orchestra.run.vpm057.stdout:Purging configuration files for radosgw (0.85-988-g7552571-1trusty) ... 2014-09-25 21:53:13,685.685 INFO:teuthology.orchestra.run.vpm057.stderr:dpkg: warning: while removing radosgw, directory '/var/lib/ceph' not empty so not removed 2014-09-25 21:53:13,688.688 INFO:teuthology.orchestra.run.vpm057.stdout:Removing ceph-test-dbg (0.85-988-g7552571-1trusty) ... 2014-09-25 21:53:13,809.809 INFO:teuthology.orchestra.run.vpm057.stdout:Removing ceph-common-dbg (0.85-988-g7552571-1trusty) ... 2014-09-25 21:53:13,844.844 INFO:teuthology.orchestra.run.vpm057.stdout:Removing ceph-test (0.85-988-g7552571-1trusty) ... 2014-09-25 21:53:13,882.882 INFO:teuthology.orchestra.run.vpm057.stdout:Removing ceph-common (0.85-988-g7552571-1trusty) ... 2014-09-25 21:53:13,901.901 INFO:teuthology.orchestra.run.vpm057.stdout:Purging configuration files for ceph-common (0.85-988-g7552571-1trusty) ... 2014-09-25 21:53:13,953.953 INFO:teuthology.orchestra.run.vpm057.stdout:Processing triggers for man-db (2.6.6-1) ... 2014-09-25 21:53:14,746.746 INFO:teuthology.orchestra.run.vpm057.stdout:Reading package lists... 2014-09-25 21:53:14,910.910 INFO:teuthology.orchestra.run.vpm057.stdout:Building dependency tree... 2014-09-25 21:53:14,911.911 INFO:teuthology.orchestra.run.vpm057.stdout:Reading state information... 2014-09-25 21:53:15,030.030 INFO:teuthology.orchestra.run.vpm057.stdout:Package 'ceph-common-dbg' is not installed, so not removed 2014-09-25 21:53:15,031.031 INFO:teuthology.orchestra.run.vpm057.stdout:The following packages were automatically installed and are no longer required: 2014-09-25 21:53:15,031.031 INFO:teuthology.orchestra.run.vpm057.stdout: btrfs-tools ceph-fs-common libaio1 libbabeltrace-ctf1 libbabeltrace1 2014-09-25 21:53:15,032.032 INFO:teuthology.orchestra.run.vpm057.stdout: libboost-program-options1.54.0 libfcgi0ldbl libgoogle-perftools4 libleveldb1 2014-09-25 21:53:15,032.032 INFO:teuthology.orchestra.run.vpm057.stdout: liblzo2-2 libradosstriper1 libreadline5 libsnappy1 libtcmalloc-minimal4 2014-09-25 21:53:15,033.033 INFO:teuthology.orchestra.run.vpm057.stdout: libunwind8 xfsprogs xml2 2014-09-25 21:53:15,033.033 INFO:teuthology.orchestra.run.vpm057.stdout:Use 'apt-get autoremove' to remove them. 2014-09-25 21:53:15,069.069 INFO:teuthology.orchestra.run.vpm057.stdout:0 upgraded, 0 newly installed, 0 to remove and 207 not upgraded. 2014-09-25 21:53:15,141.141 INFO:teuthology.orchestra.run.vpm057.stdout:Reading package lists... 2014-09-25 21:53:15,249.249 INFO:teuthology.orchestra.run.vpm057.stdout:Building dependency tree... 2014-09-25 21:53:15,250.250 INFO:teuthology.orchestra.run.vpm057.stdout:Reading state information... 2014-09-25 21:53:15,359.359 INFO:teuthology.orchestra.run.vpm057.stdout:The following packages were automatically installed and are no longer required: 2014-09-25 21:53:15,360.360 INFO:teuthology.orchestra.run.vpm057.stdout: btrfs-tools ceph-fs-common libaio1 libbabeltrace-ctf1 libbabeltrace1 2014-09-25 21:53:15,360.360 INFO:teuthology.orchestra.run.vpm057.stdout: libboost-program-options1.54.0 libfcgi0ldbl libgoogle-perftools4 libleveldb1 2014-09-25 21:53:15,361.361 INFO:teuthology.orchestra.run.vpm057.stdout: liblzo2-2 libradosstriper1 libreadline5 libsnappy1 libtcmalloc-minimal4 2014-09-25 21:53:15,361.361 INFO:teuthology.orchestra.run.vpm057.stdout: libunwind8 xfsprogs xml2 2014-09-25 21:53:15,361.361 INFO:teuthology.orchestra.run.vpm057.stdout:Use 'apt-get autoremove' to remove them. 2014-09-25 21:53:15,761.761 INFO:teuthology.orchestra.run.vpm057.stdout:The following packages will be REMOVED: 2014-09-25 21:53:15,762.762 INFO:teuthology.orchestra.run.vpm057.stdout: ceph-fuse* ceph-fuse-dbg* 2014-09-25 21:53:15,762.762 INFO:teuthology.orchestra.run.vpm057.stdout:0 upgraded, 0 newly installed, 2 to remove and 207 not upgraded. 2014-09-25 21:53:15,762.762 INFO:teuthology.orchestra.run.vpm057.stdout:After this operation, 88.2 MB disk space will be freed. 2014-09-25 21:53:15,762.762 INFO:teuthology.orchestra.run.vpm057.stdout:(Reading database ... 52581 files and directories currently installed.) 2014-09-25 21:53:15,762.762 INFO:teuthology.orchestra.run.vpm057.stdout:Removing ceph-fuse-dbg (0.85-988-g7552571-1trusty) ... 2014-09-25 21:53:15,762.762 INFO:teuthology.orchestra.run.vpm057.stdout:Removing ceph-fuse (0.85-988-g7552571-1trusty) ... 2014-09-25 21:53:15,763.763 INFO:teuthology.orchestra.run.vpm057.stdout:Processing triggers for man-db (2.6.6-1) ... 2014-09-25 21:53:16,474.474 INFO:teuthology.orchestra.run.vpm057.stdout:Reading package lists... 2014-09-25 21:53:16,600.600 INFO:teuthology.orchestra.run.vpm057.stdout:Building dependency tree... 2014-09-25 21:53:16,600.600 INFO:teuthology.orchestra.run.vpm057.stdout:Reading state information... 2014-09-25 21:53:16,702.702 INFO:teuthology.orchestra.run.vpm057.stdout:Package 'ceph-fuse-dbg' is not installed, so not removed 2014-09-25 21:53:16,702.702 INFO:teuthology.orchestra.run.vpm057.stdout:The following packages were automatically installed and are no longer required: 2014-09-25 21:53:16,703.703 INFO:teuthology.orchestra.run.vpm057.stdout: btrfs-tools ceph-fs-common libaio1 libbabeltrace-ctf1 libbabeltrace1 2014-09-25 21:53:16,703.703 INFO:teuthology.orchestra.run.vpm057.stdout: libboost-program-options1.54.0 libfcgi0ldbl libgoogle-perftools4 libleveldb1 2014-09-25 21:53:16,704.704 INFO:teuthology.orchestra.run.vpm057.stdout: liblzo2-2 libradosstriper1 libreadline5 libsnappy1 libtcmalloc-minimal4 2014-09-25 21:53:16,705.705 INFO:teuthology.orchestra.run.vpm057.stdout: libunwind8 xfsprogs xml2 2014-09-25 21:53:16,705.705 INFO:teuthology.orchestra.run.vpm057.stdout:Use 'apt-get autoremove' to remove them. 2014-09-25 21:53:16,732.732 INFO:teuthology.orchestra.run.vpm057.stdout:0 upgraded, 0 newly installed, 0 to remove and 207 not upgraded. 2014-09-25 21:53:16,792.792 INFO:teuthology.orchestra.run.vpm057.stdout:Reading package lists... 2014-09-25 21:53:16,903.903 INFO:teuthology.orchestra.run.vpm057.stdout:Building dependency tree... 2014-09-25 21:53:16,904.904 INFO:teuthology.orchestra.run.vpm057.stdout:Reading state information... 2014-09-25 21:53:17,067.067 INFO:teuthology.orchestra.run.vpm057.stdout:Package 'ceph-test' is not installed, so not removed 2014-09-25 21:53:17,068.068 INFO:teuthology.orchestra.run.vpm057.stdout:The following packages were automatically installed and are no longer required: 2014-09-25 21:53:17,069.069 INFO:teuthology.orchestra.run.vpm057.stdout: btrfs-tools ceph-fs-common libaio1 libbabeltrace-ctf1 libbabeltrace1 2014-09-25 21:53:17,069.069 INFO:teuthology.orchestra.run.vpm057.stdout: libboost-program-options1.54.0 libfcgi0ldbl libgoogle-perftools4 libleveldb1 2014-09-25 21:53:17,070.070 INFO:teuthology.orchestra.run.vpm057.stdout: liblzo2-2 libradosstriper1 libreadline5 libsnappy1 libtcmalloc-minimal4 2014-09-25 21:53:17,070.070 INFO:teuthology.orchestra.run.vpm057.stdout: libunwind8 xfsprogs xml2 2014-09-25 21:53:17,071.071 INFO:teuthology.orchestra.run.vpm057.stdout:Use 'apt-get autoremove' to remove them. 2014-09-25 21:53:17,100.100 INFO:teuthology.orchestra.run.vpm057.stdout:0 upgraded, 0 newly installed, 0 to remove and 207 not upgraded. 2014-09-25 21:53:17,115.115 INFO:teuthology.orchestra.run.vpm057.stdout:Reading package lists... 2014-09-25 21:53:17,257.257 INFO:teuthology.orchestra.run.vpm057.stdout:Building dependency tree... 2014-09-25 21:53:17,258.258 INFO:teuthology.orchestra.run.vpm057.stdout:Reading state information... 2014-09-25 21:53:17,382.382 INFO:teuthology.orchestra.run.vpm057.stdout:Package 'ceph-test-dbg' is not installed, so not removed 2014-09-25 21:53:17,382.382 INFO:teuthology.orchestra.run.vpm057.stdout:The following packages were automatically installed and are no longer required: 2014-09-25 21:53:17,383.383 INFO:teuthology.orchestra.run.vpm057.stdout: btrfs-tools ceph-fs-common libaio1 libbabeltrace-ctf1 libbabeltrace1 2014-09-25 21:53:17,383.383 INFO:teuthology.orchestra.run.vpm057.stdout: libboost-program-options1.54.0 libfcgi0ldbl libgoogle-perftools4 libleveldb1 2014-09-25 21:53:17,596.596 INFO:teuthology.orchestra.run.vpm057.stdout: liblzo2-2 libradosstriper1 libreadline5 libsnappy1 libtcmalloc-minimal4 2014-09-25 21:53:17,596.596 INFO:teuthology.orchestra.run.vpm057.stdout: libunwind8 xfsprogs xml2 2014-09-25 21:53:17,596.596 INFO:teuthology.orchestra.run.vpm057.stdout:Use 'apt-get autoremove' to remove them. 2014-09-25 21:53:17,596.596 INFO:teuthology.orchestra.run.vpm057.stdout:0 upgraded, 0 newly installed, 0 to remove and 207 not upgraded. 2014-09-25 21:53:17,596.596 INFO:teuthology.orchestra.run.vpm057.stdout:Reading package lists... 2014-09-25 21:53:17,596.596 INFO:teuthology.orchestra.run.vpm057.stdout:Building dependency tree... 2014-09-25 21:53:17,596.596 INFO:teuthology.orchestra.run.vpm057.stdout:Reading state information... 2014-09-25 21:53:17,613.613 INFO:teuthology.orchestra.run.vpm057.stdout:Package 'radosgw' is not installed, so not removed 2014-09-25 21:53:17,613.613 INFO:teuthology.orchestra.run.vpm057.stdout:The following packages were automatically installed and are no longer required: 2014-09-25 21:53:17,613.613 INFO:teuthology.orchestra.run.vpm057.stdout: btrfs-tools ceph-fs-common libaio1 libbabeltrace-ctf1 libbabeltrace1 2014-09-25 21:53:17,614.614 INFO:teuthology.orchestra.run.vpm057.stdout: libboost-program-options1.54.0 libfcgi0ldbl libgoogle-perftools4 libleveldb1 2014-09-25 21:53:17,623.623 INFO:teuthology.orchestra.run.vpm057.stdout: liblzo2-2 libradosstriper1 libreadline5 libsnappy1 libtcmalloc-minimal4 2014-09-25 21:53:17,623.623 INFO:teuthology.orchestra.run.vpm057.stdout: libunwind8 xfsprogs xml2 2014-09-25 21:53:17,623.623 INFO:teuthology.orchestra.run.vpm057.stdout:Use 'apt-get autoremove' to remove them. 2014-09-25 21:53:17,650.650 INFO:teuthology.orchestra.run.vpm057.stdout:0 upgraded, 0 newly installed, 0 to remove and 207 not upgraded. 2014-09-25 21:53:17,775.775 INFO:teuthology.orchestra.run.vpm057.stdout:Reading package lists... 2014-09-25 21:53:17,834.834 INFO:teuthology.orchestra.run.vpm057.stdout:Building dependency tree... 2014-09-25 21:53:17,835.835 INFO:teuthology.orchestra.run.vpm057.stdout:Reading state information... 2014-09-25 21:53:17,971.971 INFO:teuthology.orchestra.run.vpm057.stdout:Package 'radosgw-dbg' is not installed, so not removed 2014-09-25 21:53:17,971.971 INFO:teuthology.orchestra.run.vpm057.stdout:The following packages were automatically installed and are no longer required: 2014-09-25 21:53:17,971.971 INFO:teuthology.orchestra.run.vpm057.stdout: btrfs-tools ceph-fs-common libaio1 libbabeltrace-ctf1 libbabeltrace1 2014-09-25 21:53:17,971.971 INFO:teuthology.orchestra.run.vpm057.stdout: libboost-program-options1.54.0 libfcgi0ldbl libgoogle-perftools4 libleveldb1 2014-09-25 21:53:17,972.972 INFO:teuthology.orchestra.run.vpm057.stdout: liblzo2-2 libradosstriper1 libreadline5 libsnappy1 libtcmalloc-minimal4 2014-09-25 21:53:17,972.972 INFO:teuthology.orchestra.run.vpm057.stdout: libunwind8 xfsprogs xml2 2014-09-25 21:53:17,973.973 INFO:teuthology.orchestra.run.vpm057.stdout:Use 'apt-get autoremove' to remove them. 2014-09-25 21:53:18,020.020 INFO:teuthology.orchestra.run.vpm057.stdout:0 upgraded, 0 newly installed, 0 to remove and 207 not upgraded. 2014-09-25 21:53:18,020.020 INFO:teuthology.orchestra.run.vpm057.stdout:Reading package lists... 2014-09-25 21:53:18,162.162 INFO:teuthology.orchestra.run.vpm057.stdout:Building dependency tree... 2014-09-25 21:53:18,164.164 INFO:teuthology.orchestra.run.vpm057.stdout:Reading state information... 2014-09-25 21:53:18,267.267 INFO:teuthology.orchestra.run.vpm057.stdout:The following packages were automatically installed and are no longer required: 2014-09-25 21:53:18,268.268 INFO:teuthology.orchestra.run.vpm057.stdout: btrfs-tools ceph-fs-common libaio1 libbabeltrace-ctf1 libbabeltrace1 2014-09-25 21:53:18,269.269 INFO:teuthology.orchestra.run.vpm057.stdout: libboost-program-options1.54.0 libfcgi0ldbl libgoogle-perftools4 2014-09-25 21:53:18,270.270 INFO:teuthology.orchestra.run.vpm057.stdout: libjs-jquery libleveldb1 liblzo2-2 libradosstriper1 libreadline5 libsnappy1 2014-09-25 21:53:18,270.270 INFO:teuthology.orchestra.run.vpm057.stdout: libtcmalloc-minimal4 libunwind8 python-blinker python-flask 2014-09-25 21:53:18,271.271 INFO:teuthology.orchestra.run.vpm057.stdout: python-itsdangerous python-jinja2 python-markupsafe python-pyinotify 2014-09-25 21:53:18,271.271 INFO:teuthology.orchestra.run.vpm057.stdout: python-werkzeug xfsprogs xml2 2014-09-25 21:53:18,272.272 INFO:teuthology.orchestra.run.vpm057.stdout:Use 'apt-get autoremove' to remove them. 2014-09-25 21:53:18,290.290 INFO:teuthology.orchestra.run.vpm057.stdout:The following packages will be REMOVED: 2014-09-25 21:53:18,377.377 INFO:teuthology.orchestra.run.vpm057.stdout: python-ceph* 2014-09-25 21:53:18,419.419 INFO:teuthology.orchestra.run.vpm057.stdout:0 upgraded, 0 newly installed, 1 to remove and 207 not upgraded. 2014-09-25 21:53:18,419.419 INFO:teuthology.orchestra.run.vpm057.stdout:After this operation, 870 kB disk space will be freed. 2014-09-25 21:53:18,470.470 INFO:teuthology.orchestra.run.vpm057.stdout:(Reading database ... 52568 files and directories currently installed.) 2014-09-25 21:53:18,479.479 INFO:teuthology.orchestra.run.vpm057.stdout:Removing python-ceph (0.85-988-g7552571-1trusty) ... 2014-09-25 21:53:18,944.944 INFO:teuthology.orchestra.run.vpm057.stdout:Reading package lists... 2014-09-25 21:53:19,113.113 INFO:teuthology.orchestra.run.vpm057.stdout:Building dependency tree... 2014-09-25 21:53:19,434.434 INFO:teuthology.orchestra.run.vpm057.stdout:Reading state information... 2014-09-25 21:53:19,434.434 INFO:teuthology.orchestra.run.vpm057.stdout:The following packages were automatically installed and are no longer required: 2014-09-25 21:53:19,434.434 INFO:teuthology.orchestra.run.vpm057.stdout: btrfs-tools ceph-fs-common libaio1 libbabeltrace-ctf1 libbabeltrace1 2014-09-25 21:53:19,435.435 INFO:teuthology.orchestra.run.vpm057.stdout: libboost-program-options1.54.0 libfcgi0ldbl libgoogle-perftools4 2014-09-25 21:53:19,435.435 INFO:teuthology.orchestra.run.vpm057.stdout: libjs-jquery libleveldb1 liblzo2-2 libradosstriper1 libreadline5 libsnappy1 2014-09-25 21:53:19,435.435 INFO:teuthology.orchestra.run.vpm057.stdout: libtcmalloc-minimal4 libunwind8 python-blinker python-flask 2014-09-25 21:53:19,435.435 INFO:teuthology.orchestra.run.vpm057.stdout: python-itsdangerous python-jinja2 python-markupsafe python-pyinotify 2014-09-25 21:53:19,435.435 INFO:teuthology.orchestra.run.vpm057.stdout: python-werkzeug xfsprogs xml2 2014-09-25 21:53:19,435.435 INFO:teuthology.orchestra.run.vpm057.stdout:Use 'apt-get autoremove' to remove them. 2014-09-25 21:53:19,435.435 INFO:teuthology.orchestra.run.vpm057.stdout:The following packages will be REMOVED: 2014-09-25 21:53:19,435.435 INFO:teuthology.orchestra.run.vpm057.stdout: libcephfs-java* libcephfs-jni* libcephfs1* libcephfs1-dbg* 2014-09-25 21:53:19,435.435 INFO:teuthology.orchestra.run.vpm057.stdout:0 upgraded, 0 newly installed, 4 to remove and 207 not upgraded. 2014-09-25 21:53:19,435.435 INFO:teuthology.orchestra.run.vpm057.stdout:After this operation, 102 MB disk space will be freed. 2014-09-25 21:53:19,624.624 INFO:teuthology.orchestra.run.vpm057.stdout:(Reading database ... 52559 files and directories currently installed.) 2014-09-25 21:53:19,626.626 INFO:teuthology.orchestra.run.vpm057.stdout:Removing libcephfs-java (0.85-988-g7552571-1trusty) ... 2014-09-25 21:53:19,669.669 INFO:teuthology.orchestra.run.vpm057.stdout:Removing libcephfs-jni (0.85-988-g7552571-1trusty) ... 2014-09-25 21:53:19,713.713 INFO:teuthology.orchestra.run.vpm057.stdout:Purging configuration files for libcephfs-jni (0.85-988-g7552571-1trusty) ... 2014-09-25 21:53:19,747.747 INFO:teuthology.orchestra.run.vpm057.stdout:Removing libcephfs1-dbg (0.85-988-g7552571-1trusty) ... 2014-09-25 21:53:19,806.806 INFO:teuthology.orchestra.run.vpm057.stdout:Removing libcephfs1 (0.85-988-g7552571-1trusty) ... 2014-09-25 21:53:19,834.834 INFO:teuthology.orchestra.run.vpm057.stdout:Purging configuration files for libcephfs1 (0.85-988-g7552571-1trusty) ... 2014-09-25 21:53:19,864.864 INFO:teuthology.orchestra.run.vpm057.stdout:Processing triggers for libc-bin (2.19-0ubuntu2) ... 2014-09-25 21:53:20,505.505 INFO:teuthology.orchestra.run.vpm057.stdout:Reading package lists... 2014-09-25 21:53:20,644.644 INFO:teuthology.orchestra.run.vpm057.stdout:Building dependency tree... 2014-09-25 21:53:20,645.645 INFO:teuthology.orchestra.run.vpm057.stdout:Reading state information... 2014-09-25 21:53:20,742.742 INFO:teuthology.orchestra.run.vpm057.stdout:Package 'libcephfs1-dbg' is not installed, so not removed 2014-09-25 21:53:20,742.742 INFO:teuthology.orchestra.run.vpm057.stdout:The following packages were automatically installed and are no longer required: 2014-09-25 21:53:20,743.743 INFO:teuthology.orchestra.run.vpm057.stdout: btrfs-tools ceph-fs-common libaio1 libbabeltrace-ctf1 libbabeltrace1 2014-09-25 21:53:20,743.743 INFO:teuthology.orchestra.run.vpm057.stdout: libboost-program-options1.54.0 libfcgi0ldbl libgoogle-perftools4 2014-09-25 21:53:20,744.744 INFO:teuthology.orchestra.run.vpm057.stdout: libjs-jquery libleveldb1 liblzo2-2 libradosstriper1 libreadline5 libsnappy1 2014-09-25 21:53:20,745.745 INFO:teuthology.orchestra.run.vpm057.stdout: libtcmalloc-minimal4 libunwind8 python-blinker python-flask 2014-09-25 21:53:20,745.745 INFO:teuthology.orchestra.run.vpm057.stdout: python-itsdangerous python-jinja2 python-markupsafe python-pyinotify 2014-09-25 21:53:20,752.752 INFO:teuthology.orchestra.run.vpm057.stdout: python-werkzeug xfsprogs xml2 2014-09-25 21:53:20,752.752 INFO:teuthology.orchestra.run.vpm057.stdout:Use 'apt-get autoremove' to remove them. 2014-09-25 21:53:20,850.850 INFO:teuthology.orchestra.run.vpm057.stdout:0 upgraded, 0 newly installed, 0 to remove and 207 not upgraded. 2014-09-25 21:53:20,850.850 INFO:teuthology.orchestra.run.vpm057.stdout:Reading package lists... 2014-09-25 21:53:20,915.915 INFO:teuthology.orchestra.run.vpm057.stdout:Building dependency tree... 2014-09-25 21:53:20,917.917 INFO:teuthology.orchestra.run.vpm057.stdout:Reading state information... 2014-09-25 21:53:21,042.042 INFO:teuthology.orchestra.run.vpm057.stdout:Package 'libcephfs-java' is not installed, so not removed 2014-09-25 21:53:21,042.042 INFO:teuthology.orchestra.run.vpm057.stdout:The following packages were automatically installed and are no longer required: 2014-09-25 21:53:21,043.043 INFO:teuthology.orchestra.run.vpm057.stdout: btrfs-tools ceph-fs-common libaio1 libbabeltrace-ctf1 libbabeltrace1 2014-09-25 21:53:21,043.043 INFO:teuthology.orchestra.run.vpm057.stdout: libboost-program-options1.54.0 libfcgi0ldbl libgoogle-perftools4 2014-09-25 21:53:21,045.045 INFO:teuthology.orchestra.run.vpm057.stdout: libjs-jquery libleveldb1 liblzo2-2 libradosstriper1 libreadline5 libsnappy1 2014-09-25 21:53:21,045.045 INFO:teuthology.orchestra.run.vpm057.stdout: libtcmalloc-minimal4 libunwind8 python-blinker python-flask 2014-09-25 21:53:21,046.046 INFO:teuthology.orchestra.run.vpm057.stdout: python-itsdangerous python-jinja2 python-markupsafe python-pyinotify 2014-09-25 21:53:21,047.047 INFO:teuthology.orchestra.run.vpm057.stdout: python-werkzeug xfsprogs xml2 2014-09-25 21:53:21,047.047 INFO:teuthology.orchestra.run.vpm057.stdout:Use 'apt-get autoremove' to remove them. 2014-09-25 21:53:21,145.145 INFO:teuthology.orchestra.run.vpm057.stdout:0 upgraded, 0 newly installed, 0 to remove and 207 not upgraded. 2014-09-25 21:53:21,145.145 INFO:teuthology.orchestra.run.vpm057:Running: "dpkg -l | grep '^.HR' | awk '{print $2}' | sudo xargs --no-run-if-empty dpkg -P --force-remove-reinstreq" 2014-09-25 21:53:21,922.922 INFO:teuthology.orchestra.run.vpm057:Running: 'sudo DEBIAN_FRONTEND=noninteractive apt-get -y --force-yes -o Dpkg::Options::="--force-confdef" -o Dpkg::Options::="--force-confold" autoremove' 2014-09-25 21:53:31,362.362 DEBUG:teuthology.parallel:result is None 2014-09-25 21:53:31,363.363 INFO:teuthology.task.install:Removing ceph sources lists 2014-09-25 21:53:31,363.363 INFO:teuthology.orchestra.run.vpm057:Running: 'sudo lsb_release -is' 2014-09-25 21:53:32,270.270 DEBUG:teuthology.misc:System to be installed: Ubuntu 2014-09-25 21:53:32,270.270 INFO:teuthology.orchestra.run.vpm057:Running: 'sudo rm -f /etc/apt/sources.list.d/ceph.list && sudo apt-get update || true' 2014-09-25 21:53:32,272.272 INFO:teuthology.orchestra.run.vpm057:Running: 'sudo rm -f /etc/apt/sources.list.d/calamari.list && sudo apt-get update || true' 2014-09-25 21:53:33,424.424 INFO:teuthology.orchestra.run.vpm057.stderr:E: Could not get lock /var/lib/apt/lists/lock - open (11: Resource temporarily unavailable) 2014-09-25 21:53:33,424.424 INFO:teuthology.orchestra.run.vpm057.stderr:E: Unable to lock directory /var/lib/apt/lists/ 2014-09-25 21:53:33,425.425 DEBUG:teuthology.parallel:result is None 2014-09-25 21:53:39,566.566 DEBUG:teuthology.parallel:result is None 2014-09-25 21:53:39,567.567 INFO:teuthology.task.install:Purging /var/lib/ceph on ubuntu@vpm057.front.sepia.ceph.com 2014-09-25 21:53:39,567.567 INFO:teuthology.orchestra.run.vpm057:Running: "sudo rm -rf --one-file-system -- /var/lib/ceph || true ; test -d /var/lib/ceph && sudo find /var/lib/ceph -mindepth 1 -maxdepth 2 -type d -exec umount '{}' ';' ; sudo rm -rf --one-file-system -- /var/lib/ceph" 2014-09-25 21:53:40,353.353 DEBUG:teuthology.parallel:result is None 2014-09-25 21:53:40,353.353 DEBUG:teuthology.run_tasks:Unwinding manager internal.timer 2014-09-25 21:53:40,353.353 INFO:teuthology.task.internal:Duration was 585.362583 seconds 2014-09-25 21:53:40,354.354 DEBUG:teuthology.run_tasks:Unwinding manager internal.syslog 2014-09-25 21:53:40,354.354 DEBUG:teuthology.run_tasks:Unwinding manager internal.sudo 2014-09-25 21:53:40,354.354 INFO:teuthology.task.internal:Restoring /etc/sudoers... 2014-09-25 21:53:40,355.355 INFO:teuthology.orchestra.run.vpm057:Running: 'sudo mv -f /etc/sudoers.orig.teuthology /etc/sudoers' 2014-09-25 21:53:41,083.083 DEBUG:teuthology.run_tasks:Unwinding manager internal.coredump 2014-09-25 21:53:41,083.083 INFO:teuthology.orchestra.run.vpm057:Running: 'sudo sysctl -w kernel.core_pattern=core && rmdir --ignore-fail-on-non-empty -- /home/ubuntu/cephtest/archive/coredump' 2014-09-25 21:53:41,828.828 INFO:teuthology.orchestra.run.vpm057.stdout:kernel.core_pattern = core 2014-09-25 21:53:41,831.831 INFO:teuthology.orchestra.run.vpm057:Running: "if test '!' -e /home/ubuntu/cephtest/archive/coredump ; then echo OK ; fi" 2014-09-25 21:53:42,642.642 DEBUG:teuthology.run_tasks:Unwinding manager internal.archive 2014-09-25 21:53:42,643.643 INFO:teuthology.task.internal:Removing archive directory... 2014-09-25 21:53:42,643.643 INFO:teuthology.orchestra.run.vpm057:Running: 'rm -rf -- /home/ubuntu/cephtest/archive' 2014-09-25 21:53:43,529.529 DEBUG:teuthology.run_tasks:Unwinding manager internal.base 2014-09-25 21:53:43,529.529 INFO:teuthology.task.internal:Tidying up after the test... 2014-09-25 21:53:43,530.530 INFO:teuthology.orchestra.run.vpm057:Running: 'rmdir -- /home/ubuntu/cephtest' 2014-09-25 21:53:44,467.467 INFO:teuthology.run:Summary data: {duration: 585.3625831604004, flavor: basic, owner: loic@dachary.org, success: true} 2014-09-25 21:53:44,467.467 WARNING:teuthology.report:No job_id found; not reporting results 2014-09-25 21:53:44,467.467 INFO:teuthology.run:pass